Re: SVN Win32 Developers -- need some help

Mark Phippard wrote:
> I think we really need to address the problems of APR ICONV in our
> next release. Please see this thread for some references:
>
> http://subversion.tigris.org/servlets/BrowseList?list=dev&by=thread&from=582337
>
>
> This library has always been a problem, when there are multiple
> clients installed and they used different compilers. But now with a
> mixture of APR ICONV 0.9.x and 1.1.x out there this problem is going
> to escalate. It crashes Subversion, which means it can also crash
> clients using Subversion.
>
> Stefan King did the work of creating an initial patch that replaces
> the dependency with Win32 API's. I think we just need some consensus
> that this is what we should do, and then get a developer to do it.
>
> I think this is important and we are going to have problems if we do
> not address this for 1.5.

Patch attached.
This works very well for TSVN already.

Some things you should know about this patch:
* it returns an error if in svn_utf_cstring_to_utf8_ex2 the 'frompage'
param is not APR_DEFAULT_CHARSET or APR_LOCALE_CHARSET (or the
equivalent svn_utf_cstring_from_utf8_ex2 and 'topage'). That's because I
don't know of an API which can convert an iconv codepage string into the
define required by the Windows APIs.
* it uses calloc() and free() for memory allocation. I could have used
the apr memory pools for that, but the memory needed there is only
needed for two lines of code and can be freed after that. I think using
an apr sub-pool is much 'heavier' than a clean calloc()/free(). And
since this is Windows-Only code anyway, we don't have to use the
platform independent apr pools.

Stefan

-- 
        ___
   oo  // \\      "De Chelonian Mobile"
  (_,\/ \_/ \     TortoiseSVN
    \ \_/_\_/>    The coolest Interface to (Sub)Version Control
    /_/   \_\     http://tortoisesvn.net

Index: subversion/libsvn_subr/utf.c
===================================================================
--- subversion/libsvn_subr/utf.c (revision 25164)
+++ subversion/libsvn_subr/utf.c (working copy)
@@ -44,6 +44,7 @@
 #define SVN_APR_UTF8_CHARSET (const char*)1208
 #endif
 
+#if !APR_HAS_XLATE
 #if APR_HAS_THREADS
 static apr_thread_mutex_t *xlate_handle_mutex = NULL;
 #endif
@@ -101,10 +102,12 @@
   node->valid = FALSE;
   return APR_SUCCESS;
 }
+#endif /* !APR_HAS_XLATE */
 
 void
 svn_utf_initialize(apr_pool_t *pool)
 {
+#if APR_HAS_XLATE
   apr_pool_t *subpool;
 #if APR_HAS_THREADS
   apr_thread_mutex_t *mutex;
@@ -129,8 +132,10 @@
       apr_pool_cleanup_register(subpool, NULL, xlate_cleanup,
                                 apr_pool_cleanup_null);
     }
+#endif /* APR_HAS_XLATE */
 }
 
+#if APR_HAS_XLATE
 /* Return a unique string key based on TOPAGE and FROMPAGE. TOPAGE and
  * FROMPAGE can be any valid arguments of the same name to
  * apr_xlate_open(). Allocate the returned string in POOL. */
@@ -391,8 +396,8 @@
                                SVN_APR_UTF8_CHARSET,
                                SVN_UTF_UTON_XLATE_HANDLE, pool);
 }
+#endif /* APR_HAS_XLATE */
 
-
 /* Copy LEN bytes of SRC, converting non-ASCII and zero bytes to ?\nnn
    sequences, allocating the result in POOL. */
 static const char *
@@ -445,6 +450,7 @@
   return new_orig;
 }
 
+#if APR_HAS_XLATE
 /* Convert SRC_LENGTH bytes of SRC_DATA in NODE->handle, store the result
    in *DEST, which is allocated in POOL. */
 static svn_error_t *
@@ -546,8 +552,8 @@
 
   return SVN_NO_ERROR;
 }
+#endif /* APR_HAS_XLATE */
 
-
 /* Return APR_EINVAL if the first LEN bytes of DATA contain anything
    other than seven-bit, non-control (except for whitespace) ASCII
    characters, finding the error pool from POOL. Otherwise, return
@@ -658,6 +664,7 @@
                           const svn_stringbuf_t *src,
                           apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   xlate_handle_node_t *node;
   svn_error_t *err;
 
@@ -679,6 +686,51 @@
   put_xlate_handle_node(node, SVN_UTF_NTOU_XLATE_HANDLE, pool);
 
   return err;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t utf8len = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_ACP, 0, src->data, src->len, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_ACP, 0, src->data, src->len, widebuf,
+ (*dest)->len);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ utf8len = WideCharToMultiByte(CP_UTF8, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (utf8len == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = svn_stringbuf_ncreate("", utf8len, pool);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ utf8len = WideCharToMultiByte(CP_UTF8, 0, widebuf, widelen,
+ (*dest)->data, (*dest)->len, NULL, NULL);
+ free(widebuf);
+ if (utf8len == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }
 
 
@@ -687,6 +739,7 @@
                        const svn_string_t *src,
                        apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   svn_stringbuf_t *destbuf;
   xlate_handle_node_t *node;
   svn_error_t *err;
@@ -711,9 +764,54 @@
   put_xlate_handle_node(node, SVN_UTF_NTOU_XLATE_HANDLE, pool);
 
   return err;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t utf8len = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_ACP, 0, src->data, src->len, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_ACP, 0, src->data, src->len, widebuf,
+ (*dest)->len);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ utf8len = WideCharToMultiByte(CP_UTF8, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (utf8len == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = svn_string_ncreate("", utf8len+1, pool);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ utf8len = WideCharToMultiByte(CP_UTF8, 0, widebuf, widelen,
+ (*dest)->data, utf8len, NULL, NULL);
+ free(widebuf);
+ if (utf8len == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }
 
-
+#if APR_HAS_XLATE
 /* Common implementation for svn_utf_cstring_to_utf8,
    svn_utf_cstring_to_utf8_ex, svn_utf_cstring_from_utf8 and
    svn_utf_cstring_from_utf8_ex. Convert SRC to DEST using NODE->handle as
@@ -739,13 +837,14 @@
     }
   return SVN_NO_ERROR;
 }
+#endif /* APR_HAS_XLATE */
 
-
 svn_error_t *
 svn_utf_cstring_to_utf8(const char **dest,
                         const char *src,
                         apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   xlate_handle_node_t *node;
   svn_error_t *err;
 
@@ -756,6 +855,50 @@
   SVN_ERR(check_cstring_utf8(*dest, pool));
 
   return SVN_NO_ERROR;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t utf8len = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_ACP, 0, src, -1, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_ACP, 0, src, -1, widebuf, widelen);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ utf8len = WideCharToMultiByte(CP_UTF8, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (utf8len == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = apr_pstrmemdup(pool, "", utf8len+1);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ utf8len = WideCharToMultiByte(CP_UTF8, 0, widebuf, widelen,
+ *dest, utf8len, NULL, NULL);
+ free(widebuf);
+ if (utf8len == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }
 
 
@@ -765,6 +908,7 @@
                             const char *frompage,
                             apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   xlate_handle_node_t *node;
   svn_error_t *err;
   const char *convset_key = get_xlate_key(SVN_APR_UTF8_CHARSET, frompage,
@@ -778,6 +922,17 @@
   SVN_ERR(check_cstring_utf8(*dest, pool));
 
   return SVN_NO_ERROR;
+#else
+ /* UI clients most likely never use the frompage param but leave
+ it NULL since they don't change the thread locale they're running
+ on. Which means converting from/to the 'current locale' is always
+ ok. */
+ if ((frompage != APR_DEFAULT_CHARSET) && (frompage != APR_LOCALE_CHARSET))
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Converting to non-default or non-local charsets is not supported!\n"));
+
+ return svn_utf_cstring_to_utf8(dest, src, pool);
+#endif
 }
 
 
@@ -797,6 +952,7 @@
                             const svn_stringbuf_t *src,
                             apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   xlate_handle_node_t *node;
   svn_error_t *err;
 
@@ -818,6 +974,51 @@
   put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool);
 
   return err;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t acplen = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src->data, src->len, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src->data, src->len,
+ widebuf, widelen);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (acplen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = svn_stringbuf_ncreate("", acplen, pool);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen,
+ (*dest)->data, (*dest)->len, NULL, NULL);
+ free(widebuf);
+ if (acplen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }
 
 
@@ -826,6 +1027,7 @@
                          const svn_string_t *src,
                          apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   svn_stringbuf_t *dbuf;
   xlate_handle_node_t *node;
   svn_error_t *err;
@@ -851,6 +1053,51 @@
   put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool);
 
   return err;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t acplen = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src->data, src->len, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src->data, src->len,
+ widebuf, widelen);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (acplen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = svn_string_ncreate("", acplen, pool);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen,
+ (*dest)->data, (*dest)->len, NULL, NULL);
+ free(widebuf);
+ if (acplen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }
 
 
@@ -859,6 +1106,7 @@
                           const char *src,
                           apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   xlate_handle_node_t *node;
   svn_error_t *err;
 
@@ -869,6 +1117,50 @@
   put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool);
 
   return err;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t acplen = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src, -1, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src, -1, widebuf, widelen);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (acplen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = apr_pstrmemdup(pool, "", acplen+1);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen,
+ *dest, acplen, NULL, NULL);
+ free(widebuf);
+ if (acplen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }
 
 
@@ -878,6 +1170,7 @@
                               const char *topage,
                               apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   xlate_handle_node_t *node;
   svn_error_t *err;
   const char *convset_key = get_xlate_key(topage, SVN_APR_UTF8_CHARSET,
@@ -891,6 +1184,12 @@
   put_xlate_handle_node(node, convset_key, pool);
 
   return err;
+#else
+ if ((frompage != APR_DEFAULT_CHARSET) && (frompage != APR_LOCALE_CHARSET))
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Converting to non-default or non-local charsets is not supported!\n"));
+ return svn_utf_cstring_from_utf8(dest, src, pool);
+#endif
 }
 
 
@@ -963,6 +1262,7 @@
                                  const svn_string_t *src,
                                  apr_pool_t *pool)
 {
+#if APR_HAS_XLATE || !WIN32
   svn_stringbuf_t *dbuf;
   xlate_handle_node_t *node;
   svn_error_t *err;
@@ -988,4 +1288,49 @@
   put_xlate_handle_node(node, SVN_UTF_UTON_XLATE_HANDLE, pool);
 
   return err;
+#else
+ apr_size_t widelen = 0;
+ apr_size_t acplen = 0;
+ wchar_t * widebuf = NULL;
+
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src->data, src->len, NULL, 0);
+ if (widelen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+
+ widebuf = calloc(widelen, sizeof(wchar_t));
+ if (widebuf == NULL)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ widelen = MultiByteToWideChar(CP_UTF8, 0, src->data, src->len,
+ widebuf, widelen);
+ if (widelen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen, NULL, 0,
+ NULL, NULL);
+ if (acplen == 0)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ }
+ *dest = apr_pstrmemdup(pool, "", acplen+1);
+ if (*dest == NULL)
+ {
+ free(widebuf);
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error allocating memory during utf8 conversion\n"));
+ }
+ acplen = WideCharToMultiByte(CP_ACP, 0, widebuf, widelen,
+ *dest, acplen, NULL, NULL);
+ free(widebuf);
+ if (acplen == 0)
+ return svn_error_createf(APR_EINVAL, NULL,
+ _("Error converting string to utf8\n"));
+ return SVN_NO_ERROR;
+#endif
 }

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@subversion.tigris.org
For additional commands, e-mail: dev-help@subversion.tigris.org
Received on Mon Jun 4 21:45:03 2007