charenc.C

/*
 * This file is part of the "Archon" framework.
 * (http://files3d.sourceforge.net)
 *
 * Copyright © 2002 by Kristian Spangsege and Brian Kristiansen.
 *
 * Permission to use, copy, modify, and distribute this software and
 * its documentation under the terms of the GNU General Public License is
 * hereby granted. No representations are made about the suitability of
 * this software for any purpose. It is provided "as is" without express
 * or implied warranty. See the GNU General Public License
 * (http://www.gnu.org/copyleft/gpl.html) for more details.
 *
 * The characters in this file are ISO8859-1 encoded.
 *
 * The documentation in this file is in "Doxygen" style
 * (http://www.doxygen.org).
 */

#include <errno.h>
#include <string.h>
#include <iconv.h>
#include <iostream>

#include <archon/util/exception.H>
#include <archon/util/text.H>
#include <archon/util/charenc.H>

using namespace std;

namespace
{
  inline bool isLittleEndianArch()
  {
    short s = 1;
    return *reinterpret_cast<char *>(&s);
  }

  inline string getUcsEncoding()
  {
      if(sizeof(wchar_t) == 2) return isLittleEndianArch() ? "UCS-2LE" : "UCS-2BE";
      if(sizeof(wchar_t) == 4) return isLittleEndianArch() ? "UCS-4LE" : "UCS-4BE";
      ARCHON_THROW1(Archon::Utilities::ResourceException,
                    "Unsupported number of bits in wchar_t");
  }
}

namespace Archon
{
  namespace Utilities
  {
    const string CharEnc::US_ASCII     = "US-ASCII";
    const string CharEnc::ISO_8859_1   = "ISO-8859-1";
    const string CharEnc::ISO_8859_15  = "ISO-8859-15";
    const string CharEnc::UTF_8        = "UTF-8";
    const string CharEnc::UTF_16LE     = "UTF-16LE";
    const string CharEnc::UTF_16BE     = "UTF-16BE";
    const string CharEnc::UTF_32LE     = "UTF-32LE";
    const string CharEnc::UTF_32BE     = "UTF-32BE";
    const string CharEnc::WINDOWS_1252 = "WINDOWS-1252";

    string CharEnc::encode(wstring s, string encoding, bool fail) throw(TranscodeException)
    {
      if(s.empty()) return string();

      CharEnc transcoder(getUcsEncoding(), encoding);
      const char *inbuf  = reinterpret_cast<const char *>(s.data());
      size_t inbytesleft = s.size()*sizeof(wchar_t);
      const size_t buffersize = 512;
      char buffer[buffersize];

      string t;
      while(inbytesleft)
      {
        char *outbuf = buffer;
        size_t outbytesleft = buffersize;       
        transcoder.transcode(inbuf, inbytesleft, outbuf, outbytesleft, fail);
        t.append(buffer, buffersize - outbytesleft);
      }

      return t;
    }

    wstring CharEnc::decode(string s, string encoding, bool fail) throw(TranscodeException)
    {
      if(s.empty()) return wstring();

      CharEnc transcoder(encoding, getUcsEncoding());
      const char *inbuf  = s.data();
      size_t inbytesleft = s.size();
      const size_t buffersize = 512;
      char buffer[buffersize];

      wstring t;
      while(inbytesleft)
      {
        char *outbuf = buffer;
        size_t outbytesleft = buffersize;       
        transcoder.transcode(inbuf, inbytesleft, outbuf, outbytesleft, fail);
        t.append(reinterpret_cast<wchar_t *>(buffer), (buffersize - outbytesleft)/sizeof(wchar_t));
      }

      return t;
    }

    string CharEnc::transcode(string s, string fromEnc, string toEnc, bool fail) throw(TranscodeException)
    {
      if(s.empty()) return string();

      CharEnc transcoder(fromEnc, toEnc);
      const char *inbuf  = s.data();
      size_t inbytesleft = s.size();
      const size_t buffersize = 512;
      char buffer[buffersize];

      string t;
      while(inbytesleft)
      {
        char *outbuf = buffer;
        size_t outbytesleft = buffersize;       
        transcoder.transcode(inbuf, inbytesleft, outbuf, outbytesleft, fail);
        t.append(buffer, buffersize - outbytesleft);
      }

      return t;
    }

    CharEnc::CharEnc(string fromEnc, string toEnc):
      targetEncoding(toEnc), needReplacementCharacter(true)
    {
      if(numeric_limits<unsigned char>::digits != 8)
        ARCHON_THROW1(ResourceException,
                      "Unsupported number of bits in char: " +
                      Text::toString(numeric_limits<unsigned char>::digits));
      iconvState = iconv_open(toEnc.c_str(), fromEnc.c_str());
      if(iconvState==reinterpret_cast<iconv_t>(-1))
        ARCHON_THROW1(ResourceException,
                      string("Unexpected error from 'iconv_open': ") +
                      strerror(errno));
    }

    CharEnc::~CharEnc()
    {
      iconv_close(iconvState);
    }

    void CharEnc::transcode(const char *&in, size_t &inbytes, char *&out, size_t &outbytes, bool fail) throw(TranscodeException)
    {
      char *inbuf = const_cast<char *>(in);
      size_t inbytesleft = inbytes;
      char *outbuf = out;
      size_t outbytesleft = outbytes;

      while(inbytesleft)
      {
        size_t n = iconv(iconvState, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
        if(n==static_cast<size_t>(-1))
        {
          if(errno == E2BIG || errno == EINVAL) break;

          if(fail) ARCHON_THROW(TranscodeException);

          if(needReplacementCharacter)
          {
            try
            {
              replacementCharacter = CharEnc::transcode("\xEF\xBF\xBD", UTF_8, targetEncoding, true);
            }
            catch(TranscodeException &)
            {
              try
              {
                replacementCharacter = CharEnc::transcode("?", US_ASCII, targetEncoding, true);
              }
              catch(TranscodeException &)
              {
                try
                {
                  replacementCharacter = CharEnc::transcode(" ", US_ASCII, targetEncoding, true);
                }
                catch(TranscodeException &)
                {
                  replacementCharacter = "";
                }
              }
            }
            needReplacementCharacter = false;
          }

          n = replacementCharacter.size();
          if(outbytesleft < n) break;
          if(n) replacementCharacter.copy(outbuf, string::npos);
          outbytesleft -= n;
          outbuf += n;
          --inbytesleft;
          ++inbuf;
        }
      }

      in       = inbuf;
      inbytes  = inbytesleft;
      out      = outbuf;
      outbytes = outbytesleft;
    }
  }
}

---