Index: ./configure.in =================================================================== --- ./configure.in +++ ./configure.in Fri May 17 17:19:18 2002 @@ -360,6 +360,16 @@ AC_SUBST(SVN_RA_LIB_LINK) +dnl descide whether we want to activate real UTF-8 translations +AC_ARG_ENABLE(utf8, +[ --enable-utf8 Turn on real UTF-8 strings], +[ + if test "$enableval" = "yes"; then + enable_utf8=yes + CFLAGS="$CFLAGS -DSVN_UTF8"; + fi +]) + dnl Pass some config data ---------------------------- AC_SUBST(abs_builddir) Index: ./subversion/include/svn_utf.h =================================================================== --- /dev/null Fri May 17 18:45:52 2002 +++ ./subversion/include/svn_utf.h Fri May 17 17:58:07 2002 @@ -0,0 +1,64 @@ +/* svn_utf.h: UTF-8 code shared by various Subversion libraries. + * + * ==================================================================== + * Copyright (c) 2000-2002 CollabNet. All rights reserved. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://subversion.tigris.org/license-1.html. + * If newer versions of this license are posted there, you may use a + * newer version instead, at your option. + * + * This software consists of voluntary contributions made by many + * individuals. For exact contribution history, see the revision + * history and logs, available at http://subversion.tigris.org/. + * ==================================================================== + */ + + + +#ifndef SVN_UTF_H +#define SVN_UTF_H + +#include "svn_error.h" +#include "svn_delta.h" + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/*** UTF-8 conversions ***/ + +/* Convert a stringbuf from native character encoding to UTF-8 */ +svn_error_t *svn_utf_stringbuf_to_utf8 (svn_stringbuf_t *src, + svn_stringbuf_t **dest, + apr_pool_t *pool); + +/* Convert a stringbuf from UTF-8 to native character encoding */ +svn_error_t *svn_utf_stringbuf_from_utf8 (svn_stringbuf_t *src, + svn_stringbuf_t **dest, + apr_pool_t *pool); + +/* Convert an UTF-8 C string to a stringbuf with native character encoding */ +svn_error_t *svn_utf_stringbuf_from_utf8_cstring (const char *src, + svn_stringbuf_t **dest, + apr_pool_t *pool); + +/* Convert a C string from UTF-8 to native character encoding */ +/* This function is for error message printing only... */ +char *svn_utf_utf8_to_native (const char *utf8_string, + char *buf, apr_size_t bufsize); + + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SVN_XML_H */ + +/* ---------------------------------------------------------------- + * local variables: + * eval: (load-file "../../tools/dev/svn-dev.el") + * end: + */ Index: ./subversion/libsvn_subr/io.c =================================================================== --- ./subversion/libsvn_subr/io.c +++ ./subversion/libsvn_subr/io.c Fri May 17 17:11:33 2002 @@ -36,6 +36,7 @@ #include "svn_io.h" #include "svn_base64.h" #include "svn_pools.h" +#include "svn_utf.h" #include "svn_private_config.h" /* for SVN_CLIENT_DIFF */ @@ -55,11 +56,14 @@ { apr_finfo_t finfo; apr_status_t apr_err; + svn_stringbuf_t *pathbuf_native; if (path[0] == '\0') path = "."; - apr_err = apr_stat (&finfo, path, APR_FINFO_MIN, pool); + SVN_ERR (svn_utf_stringbuf_from_utf8_cstring (path, &pathbuf_native, pool)); + + apr_err = apr_stat (&finfo, pathbuf_native->data, APR_FINFO_MIN, pool); if (apr_err && !APR_STATUS_IS_ENOENT(apr_err)) return svn_error_createf @@ -335,8 +339,11 @@ { apr_finfo_t finfo; apr_status_t apr_err; + svn_stringbuf_t *path_native; + + SVN_ERR (svn_utf_stringbuf_from_utf8 (path, &path_native, pool)); - apr_err = apr_stat (&finfo, path->data, APR_FINFO_MIN, pool); + apr_err = apr_stat (&finfo, path_native->data, APR_FINFO_MIN, pool); if (apr_err) return svn_error_createf (apr_err, 0, NULL, pool, @@ -1306,6 +1313,7 @@ apr_status_t apr_err; unsigned char block[1024]; apr_size_t amt_read = sizeof (block); + svn_stringbuf_t *path_native; /* Default return value is NULL. */ @@ -1318,7 +1326,9 @@ "Can't detect mimetype of non-file '%s'", file); - apr_err = apr_file_open (&fh, file, APR_READ, 0, pool); + SVN_ERR (svn_utf_stringbuf_from_utf8_cstring (file, &path_native, pool)); + + apr_err = apr_file_open (&fh, path_native->data, APR_READ, 0, pool); if (apr_err) return svn_error_createf (apr_err, 0, NULL, pool, "svn_io_detect_mimetype: error opening '%s'", Index: ./subversion/libsvn_subr/svn_error.c =================================================================== --- ./subversion/libsvn_subr/svn_error.c +++ ./subversion/libsvn_subr/svn_error.c Thu May 16 17:29:49 2002 @@ -29,6 +29,7 @@ #include "svn_pools.h" #include "svn_error.h" #include "svn_io.h" +#include "svn_utf.h" /* Key for the error pool itself. */ static const char SVN_ERROR_POOL[] = "svn-error-pool"; @@ -314,14 +315,16 @@ void svn_handle_error (svn_error_t *err, FILE *stream, svn_boolean_t fatal) { - char buf[200]; + char buf[200], buf2[200]; /* Pretty-print the error */ /* Note: we can also log errors here someday. */ #ifdef SVN_DEBUG if (err->file) - fprintf (stream, "\n%s:%ld\n", err->file, err->line); + fprintf (stream, "\n%s:%ld\n", + svn_utf_utf8_to_native (err->file, buf2, sizeof (buf2)), + err->line); else fprintf (stream, "\n%s\n", SVN_FILE_LINE_UNDEFINED); #else @@ -332,7 +335,9 @@ if ((err->apr_err > APR_OS_START_USEERR) && (err->apr_err <= APR_OS_START_CANONERR)) fprintf (stream, "svn_error: #%d : <%s>\n", err->apr_err, - svn_strerror (err->apr_err, buf, sizeof (buf))); + svn_utf_utf8_to_native (svn_strerror (err->apr_err, + buf, sizeof (buf)), + buf2, sizeof(buf2))); /* Otherwise, this must be an APR error code. */ else @@ -342,7 +347,8 @@ apr_strerror (err->apr_err, buf, sizeof(buf))); if (err->message) - fprintf (stream, " %s", err->message); + fprintf (stream, " %s", + svn_utf_utf8_to_native (err->message, buf2, sizeof(buf2))); fputc ('\n', stream); fflush (stream); @@ -361,12 +367,22 @@ { va_list ap; + apr_pool_t *pool = svn_pool_create (NULL); + svn_stringbuf_t *msg, *msg_utf8; + svn_error_t *err; + va_start (ap, fmt); - vfprintf (stderr, fmt, ap); + msg_utf8 = svn_stringbuf_create (apr_pvsprintf (pool, fmt, ap), pool); va_end (ap); - fprintf (stderr, "\n"); + err = svn_utf_stringbuf_from_utf8 (msg_utf8, &msg, pool); + if (err) + svn_handle_error (err, stderr, FALSE); + else + fprintf (stderr, "%s\n", msg->data); fflush (stderr); + + svn_pool_destroy (pool); } Index: ./subversion/libsvn_subr/utf.c =================================================================== --- /dev/null Fri May 17 18:46:07 2002 +++ ./subversion/libsvn_subr/utf.c Fri May 17 18:00:53 2002 @@ -0,0 +1,263 @@ +/* + * utf.c: UTF-8 helper code shared among the Subversion libraries. + * + * ==================================================================== + * Copyright (c) 2000-2002 CollabNet. All rights reserved. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://subversion.tigris.org/license-1.html. + * If newer versions of this license are posted there, you may use a + * newer version instead, at your option. + * + * This software consists of voluntary contributions made by many + * individuals. For exact contribution history, see the revision + * history and logs, available at http://subversion.tigris.org/. + * ==================================================================== + */ + + + +#include +#include +#include +#include + +#include + +#include "svn_string.h" +#include "svn_error.h" +#include "svn_pools.h" +#include "svn_utf.h" + + +#ifdef SVN_UTF8 + +/* Figure out what the name of the native character set is, so + that we can pass it to iconv_open. */ +static const char * +find_native_charset (void) +{ + /* First try $LC_CTYPE */ + const char *dot, *ctype = getenv ("LC_CTYPE"); + + /* Skip anything before the dot, if any */ + if (ctype != NULL && (dot = strchr (ctype, '.'))) + ctype = dot + 1; + + if (ctype != NULL && !strcmp (ctype, "iso_8859_1")) + ctype = "ISO8859-1"; + + if (ctype == NULL || ! *ctype) { + /* Fall back to US-ASCII */ + ctype = "646"; + } + + return ctype; +} + +/* Return the iconv handle for converting native characters to UTF-8. + Create one if it doesn't exist. */ +static iconv_t +get_ntou_iconv_handle (void) +{ + static iconv_t cd = (iconv_t)-1; + + /* If we already have a handle, just return it. */ + if (cd != (iconv_t)-1) + return cd; + + /* Try to create one. Also save it for later. */ + return cd = iconv_open("UTF-8", find_native_charset()); +} + +/* Return the iconv handle for converting UTF-8 to native characters. + Create one if it doesn't exist. */ +static iconv_t +get_uton_iconv_handle (void) +{ + static iconv_t cd = (iconv_t)-1; + + /* If we already have a handle, just return it. */ + if (cd != (iconv_t)-1) + return cd; + + /* Try to create one. Also save it for later. */ + return cd = iconv_open(find_native_charset(), "UTF-8"); +} + + +static svn_error_t * +svn_utf_convert_to_stringbuf (iconv_t cd, + const char *src_data, apr_size_t src_length, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + /* 2 bytes per character will be enough in most cases. + If not, we'll make a larger buffer and try again. */ + apr_size_t buflen = src_length * 2 + 1; + + if (cd == (iconv_t)-1) + return svn_error_create (0, errno, NULL, pool, "recoding string"); + + do { + + char *destbuf = apr_palloc (pool, buflen); + + /* Set up state variables for iconv */ + const char *srcptr = src_data; + char *destptr = destbuf; + size_t srclen = src_length; + size_t destlen = buflen; + + /* Attempt the conversion */ + if (iconv(cd, &srcptr, &srclen, &destptr, &destlen) != (size_t)-1) { + + /* Conversion succeeded. Return buffer */ + *dest = svn_stringbuf_ncreate (destbuf, buflen - destlen, pool); + + return SVN_NO_ERROR; + } + + /* In case we got here because the buffer was too small, + double the size for the next iteration... */ + buflen *= 2; + + } while (errno == E2BIG); + + /* Some other error occured. */ + return svn_error_create (0, errno, NULL, pool, "recoding string"); +} + + +svn_error_t * +svn_utf_stringbuf_to_utf8 (svn_stringbuf_t *src, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + /* Get a converter from the native character encoding to UTF-8 */ + iconv_t cd = get_ntou_iconv_handle (); + + return svn_utf_convert_to_stringbuf (cd, src->data, src->len, dest, pool); +} + +svn_error_t * +svn_utf_stringbuf_from_utf8 (svn_stringbuf_t *src, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + /* Get a converter from UTF-8 to the native character encoding */ + iconv_t cd = get_uton_iconv_handle (); + + return svn_utf_convert_to_stringbuf (cd, src->data, src->len, dest, pool); +} + +svn_error_t * +svn_utf_stringbuf_from_utf8_cstring (const char *src, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + /* Get a converter from UTF-8 to the native character encoding */ + iconv_t cd = get_uton_iconv_handle (); + + return svn_utf_convert_to_stringbuf (cd, src, strlen(src), dest, pool); +} + + +char * +svn_utf_utf8_to_native (const char *utf8_string, + char *buf, apr_size_t bufsize) +{ + /* Set up state variables for iconv */ + const char *srcptr = utf8_string; + char *destptr = buf; + size_t srclen = strlen (utf8_string); + size_t destlen = bufsize-1; + + /* Get a converter from UTF-8 to the native character encoding */ + iconv_t cd = get_uton_iconv_handle (); + + if (cd == (iconv_t)-1) + return NULL; + + /* Attempt the conversion */ + if (iconv(cd, &srcptr, &srclen, &destptr, &destlen) != (size_t)-1) { + /* Conversion succeeded. Zero-terminate and return buffer */ + *destptr = '\0'; + return buf; + } + + return NULL; +} + +#else + +static svn_error_t * +check_non_ascii (const char *data, apr_size_t len, apr_pool_t *pool) +{ + for (; len > 0; --len, data++) + if (*(unsigned char *)data >= 128) + return svn_error_create (0, 0, NULL, pool, + "non-ascii characters detected, " + "please compile with --enable-utf8"); + return SVN_NO_ERROR; +} + +svn_error_t * +svn_utf_stringbuf_to_utf8 (svn_stringbuf_t *src, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + SVN_ERR (check_non_ascii (src->data, src->len, pool)); + *dest = src; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_utf_stringbuf_from_utf8 (svn_stringbuf_t *src, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + SVN_ERR (check_non_ascii (src->data, src->len, pool)); + *dest = src; + return SVN_NO_ERROR; +} + +svn_error_t * +svn_utf_stringbuf_from_utf8_cstring (const char *src, + svn_stringbuf_t **dest, + apr_pool_t *pool) +{ + SVN_ERR (check_non_ascii (src, strlen(src), pool)); + *dest = svn_stringbuf_create (src, pool); + return SVN_NO_ERROR; +} + +char * +svn_utf_utf8_to_native (const char *utf8_string, + char *buf, apr_size_t bufsize) +{ + int i; + + /* Just replace non-ASCII characters with '?' here... */ + + for (i=0; i= 192) + /* First octet of a multibyte sequence */ + buf[i++] = '?'; + + buf[i>=bufsize? bufsize-1 : i] = '\0'; + return buf; +} + +#endif /* SVN_UTF8 */ + + +/* + * local variables: + * eval: (load-file "../../tools/dev/svn-dev.el") + * end: + */ Index: ./subversion/clients/cmdline/status.c =================================================================== --- ./subversion/clients/cmdline/status.c +++ ./subversion/clients/cmdline/status.c Thu May 16 19:55:42 2002 @@ -26,6 +26,7 @@ #include "svn_sorts.h" #include "svn_wc.h" #include "svn_string.h" +#include "svn_utf.h" #include "cl.h" @@ -241,6 +242,8 @@ for (i = 0; i < statusarray->nelts; i++) { const svn_item_t *item; + svn_stringbuf_t *path; + svn_error_t *err; item = &APR_ARRAY_IDX(statusarray, i, const svn_item_t); status = item->value; @@ -248,10 +251,14 @@ if ((skip_unrecognized) && (! status->entry)) continue; + err = svn_utf_stringbuf_from_utf8_cstring (item->key, &path, pool); + if (err) + svn_handle_error (err, stderr, FALSE); + if (detailed) - print_long_format (item->key, show_last_committed, status); + print_long_format (path->data, show_last_committed, status); else - print_short_format (item->key, status); + print_short_format (path->data, status); } /* If printing in detailed format, we might have a head revision to Index: ./subversion/clients/cmdline/add-cmd.c =================================================================== --- ./subversion/clients/cmdline/add-cmd.c +++ ./subversion/clients/cmdline/add-cmd.c Thu May 16 19:44:03 2002 @@ -29,6 +29,7 @@ #include "svn_delta.h" #include "svn_error.h" #include "svn_pools.h" +#include "svn_utf.h" #include "cl.h" @@ -54,7 +55,11 @@ for (i = 0; i < targets->nelts; i++) { svn_stringbuf_t *target = ((svn_stringbuf_t **) (targets->elts))[i]; - err = svn_client_add (target, recursive, + svn_stringbuf_t *target_utf8; + + SVN_ERR (svn_utf_stringbuf_to_utf8 (target, &target_utf8, subpool)); + + err = svn_client_add (target_utf8, recursive, SVN_CL_NOTIFY(opt_state), svn_cl__make_notify_baton (subpool), subpool); Index: ./subversion/clients/cmdline/status-cmd.c =================================================================== --- ./subversion/clients/cmdline/status-cmd.c +++ ./subversion/clients/cmdline/status-cmd.c Thu May 16 19:44:00 2002 @@ -28,6 +28,7 @@ #include "svn_path.h" #include "svn_delta.h" #include "svn_error.h" +#include "svn_utf.h" #include "cl.h" @@ -55,6 +56,9 @@ for (i = 0; i < targets->nelts; i++) { svn_stringbuf_t *target = ((svn_stringbuf_t **) (targets->elts))[i]; + svn_stringbuf_t *target_utf8; + + SVN_ERR (svn_utf_stringbuf_to_utf8 (target, &target_utf8, pool)); /* Retrieve a hash of status structures with the information requested by the user. @@ -62,7 +66,8 @@ svn_client_status directly understands the three commandline switches (-n, -u, -[vV]) : */ - SVN_ERR (svn_client_status (&statushash, &youngest, target, auth_baton, + SVN_ERR (svn_client_status (&statushash, &youngest, target_utf8, + auth_baton, opt_state->nonrecursive ? 0 : 1, opt_state->verbose, opt_state->update, Index: ./subversion/clients/cmdline/feedback.c =================================================================== --- ./subversion/clients/cmdline/feedback.c +++ ./subversion/clients/cmdline/feedback.c Thu May 16 19:22:28 2002 @@ -29,6 +29,7 @@ #include #include "svn_pools.h" +#include "svn_utf.h" #include "cl.h" @@ -40,21 +41,16 @@ static void -notify_added (void *baton, const char *path) +notify_added (const char *path, const char *path_utf8, + svn_stringbuf_t *spath, apr_pool_t *pool) { - struct notify_baton *nb = (struct notify_baton *) baton; - - /* the pool (BATON) is typically the global pool; don't keep filling it */ - apr_pool_t *subpool = svn_pool_create (nb->pool); - - svn_stringbuf_t *spath = svn_stringbuf_create (path, subpool); svn_wc_entry_t *entry; svn_error_t *err; const char *type = " "; /* fill with "binary" if binary, etc */ /* ### this sucks. we have to open/parse the entries file to get this ### information. when adding thousands of files, this blows... */ - err = svn_wc_entry (&entry, spath, FALSE, subpool); + err = svn_wc_entry (&entry, spath, FALSE, pool); if (err) { printf ("WARNING: error fetching entry for %s\n", path); @@ -71,7 +67,7 @@ const svn_string_t *value; /* ### and again: open/parse a properties file. urk... */ - err = svn_wc_prop_get (&value, SVN_PROP_MIME_TYPE, path, subpool); + err = svn_wc_prop_get (&value, SVN_PROP_MIME_TYPE, path_utf8, pool); if (err) { printf ("WARNING: error fetching %s property for %s\n", @@ -88,7 +84,7 @@ printf ("A %s %s\n", type, path); done: - svn_pool_destroy (subpool); + ; } @@ -114,55 +110,74 @@ svn_wc_notify_action_t action, const char *path) { + struct notify_baton *nb = (struct notify_baton *) baton; + + /* the pool (BATON) is typically the global pool; don't keep filling it */ + apr_pool_t *subpool = svn_pool_create (nb->pool); + + svn_stringbuf_t *spath = svn_stringbuf_create (path, subpool); + svn_stringbuf_t *spath_native; + svn_error_t *err; + + err = svn_utf_stringbuf_from_utf8 (spath, &spath_native, subpool); + if (err) + { + printf ("WARNING: error decoding UTF-8 for ?\n"); + goto done; + } + switch (action) { case svn_wc_notify_add: - notify_added (baton, path); - return; + notify_added (spath_native->data, path, spath, subpool); + break; case svn_wc_notify_delete: - printf ("D %s\n", path); - return; + printf ("D %s\n", spath_native->data); + break; case svn_wc_notify_restore: - printf ("Restored %s\n", path); - return; + printf ("Restored %s\n", spath_native->data); + break; case svn_wc_notify_revert: - printf ("Reverted %s\n", path); - return; + printf ("Reverted %s\n", spath_native->data); + break; case svn_wc_notify_resolve: - printf ("Resolved conflicted state of %s\n", path); - return; + printf ("Resolved conflicted state of %s\n", spath_native->data); + break; case svn_wc_notify_update: - printf ("U %s\n", path); - return; + printf ("U %s\n", spath_native->data); + break; case svn_wc_notify_commit_modified: - printf ("Sending %s\n", path); - return; + printf ("Sending %s\n", spath_native->data); + break; case svn_wc_notify_commit_added: - printf ("Adding %s\n", path); - return; + printf ("Adding %s\n", spath_native->data); + break; case svn_wc_notify_commit_deleted: - printf ("Deleting %s\n", path); - return; + printf ("Deleting %s\n", spath_native->data); + break; case svn_wc_notify_commit_replaced: - printf ("Replacing %s\n", path); - return; + printf ("Replacing %s\n", spath_native->data); + break; case svn_wc_notify_commit_postfix_txdelta: notify_commit_postfix_txdelta (baton, path); - return; + break; default: break; } + + done: + svn_pool_destroy (subpool); }