Add facet based interface

5 files changed, 288 insertions, 27 deletions

diff --git a/Makefile b/Makefile
index b66c17e..5d64631 100644
--- a/Makefile
+++ b/Makefile
@@ -60,9 +60,10 @@ endif
 
 SRC=\
     src/recode.cpp \
+    src/validate.cpp \
     src/test-unicode.cpp
 
-all: src/recode src/test-unicode
+all: src/recode src/test-unicode src/validate
 
 test: src/test-unicode
 	src/test-unicode
@@ -70,6 +71,9 @@ test: src/test-unicode
 src/recode: src/recode.o dep
 	$(CXX) $(LDFLAGS) $< $(LDLIBS) $(LIBS) -o $@
 
+src/validate: src/validate.o dep
+	$(CXX) $(LDFLAGS) $< $(LDLIBS) $(LIBS) -o $@
+
 src/test-unicode: src/test-unicode.o dep
 	$(CXX) $(LDFLAGS) $< $(LDLIBS) $(LIBS) -o $@
 
@@ -82,7 +86,7 @@ dep: $(SRC:.cpp=.d)
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 clean:
-	-rm -f src/recode src/test-unicode
+	-rm -f src/recode src/test-unicode src/validate
 	-rm -rf result
 	-find . -name '*.o' -o -name '*.d' -o -name '*.gcno' -o -name '*.gcda' | xargs rm -f
 
diff --git a/debian/control b/debian/control
index 1572512..9d31022 100644
--- a/debian/control
+++ b/debian/control
@@ -19,3 +19,13 @@ Description: Unicode conversion library
   - Additional support for ISO-8859-15
   - Tested on Debian 10, Ubuntu 2004, Ubuntu 2010
   - C++17 and C++20 compatible
+
+Package: unicode-tools
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Homepage: http://www.reichwein.it/unicode/
+Description: Unicode conversion tools
+ unicode-tools is a collection of tools for Unicode file conversion:
+ .
+  - unicode-recode: Recode Unicode or ISO-8859 file
+  - unicode-validate: Check file for Unicode compliance
diff --git a/include/unicode.h b/include/unicode.h
index f31cbac..4b676bf 100644
--- a/include/unicode.h
+++ b/include/unicode.h
@@ -3,8 +3,10 @@
 #pragma once
 
 #include <algorithm>
+#include <memory>
 #include <stdexcept>
 #include <string>
+#include <unordered_map>
 
 #ifdef __cpp_char8_t
 // char8_t available
@@ -31,7 +33,7 @@ namespace unicode::detail {
  template<typename T>
  struct utf_iterator
  {
-  typedef char32_t value_type;
+  typedef T value_type;
   typedef char32_t& reference;
   typedef std::basic_string<T> string_type;
 
@@ -201,6 +203,7 @@ namespace unicode::detail {
    return value;
   }
 
+ private:
   typename string_type::const_iterator iterator;
   typename string_type::const_iterator end_iterator;
 
@@ -211,13 +214,14 @@ namespace unicode::detail {
  template<typename T>
  struct utf_back_insert_iterator
  {
+  typedef T value_type;
   typedef std::basic_string<T> string_type;
   typedef utf_back_insert_iterator& reference;
 
   utf_back_insert_iterator(string_type& s): s(s) {}
 
   // no-op
-  utf_back_insert_iterator& operator++()
+  reference operator++()
   {
    return *this;
   }
@@ -302,39 +306,220 @@ namespace unicode::detail {
    return *this;
   }
 
+ private:
   typename utf_back_insert_iterator::string_type& s;
  };
 
- template<typename T>
- utf_back_insert_iterator<T> utf_back_inserter(std::basic_string<T>& s)
- {
-  return utf_back_insert_iterator<T>(s);
+ typedef std::unordered_map<utf8_t, char32_t> iso_map_type;
+ typedef std::unordered_map<char32_t, utf8_t> iso_map_type_reverse;
+
+ // ISO-8859-1 is lower 8-bit of Unicode, so no exceptions necessary
+ iso_map_type iso_8859_1_map;
+
+ // ISO-8859-15 is lower 8-bit of Unicode, except for:
+ iso_map_type iso_8859_15_map {
+  { '\xA4', U'\u20AC' }, // €
+  { '\xA6', U'\u0160' }, // Š
+  { '\xA8', U'\u0161' }, // š
+  { '\xB4', U'\u017D' }, // Ž
+  { '\xB8', U'\u017E' }, // ž
+  { '\xBC', U'\u0152' }, // Œ
+  { '\xBD', U'\u0153' }, // œ
+  { '\xBE', U'\u0178' }, // Ÿ
+ };
+
+ iso_map_type_reverse reverse_iso_map(const iso_map_type& map) {
+  iso_map_type_reverse result;
+  std::for_each(map.cbegin(), map.cend(),
+                [&](const iso_map_type::value_type& pair)
+                 {
+                  result.emplace(pair.second, pair.first);
+                 });
+  return result;
  }
 
- template<typename T>
- utf_iterator<T> utf_begin(const std::basic_string<T>& s)
+ iso_map_type_reverse iso_8859_15_map_reverse { reverse_iso_map(iso_8859_15_map) };
+ iso_map_type_reverse iso_8859_1_map_reverse { reverse_iso_map(iso_8859_1_map) };
+
+} // namespace unicode::detail
+
+namespace unicode {
+
+ using namespace detail;
+
+ template<unicode::detail::iso_map_type& Map=iso_8859_1_map>
+ struct iso_iterator {
+  typedef char32_t value_type;
+  typedef char32_t& reference;
+  typedef std::basic_string<utf8_t>::const_iterator iterator;
+
+  iso_iterator(const iterator& it): m_it(it) {}
+
+  // pre-increment
+  iso_iterator& operator++()
+  {
+   ++m_it;
+   return *this;
+  }
+
+  bool operator!=(const iso_iterator& other) const
+  {
+   return m_it != other.m_it;
+  }
+
+  // return reference?
+  value_type operator*()
+  {
+   utf8_t value{*m_it};
+
+   if constexpr(std::addressof(Map) != std::addressof(iso_8859_1_map)) // mapping of 128 <= x <= 255 needed
+   {
+    auto it{Map.find(value)};
+    if (it != Map.end())
+     return it->second;
+   }
+   return static_cast<value_type>(static_cast<uint8_t>(value));
+  }
+
+ private:
+  iterator m_it;
+ };
+
+ template<unicode::detail::iso_map_type_reverse& Map=iso_8859_1_map_reverse>
+ struct iso_back_insert_iterator {
+  typedef iso_back_insert_iterator& reference;
+  typedef std::basic_string<utf8_t> string_type;
+  
+  iso_back_insert_iterator(string_type& s): s(s) {}
+
+  // no-op
+  reference operator++()
+  {
+   return *this;
+  }
+
+  // support *x = value, together with operator=()
+  reference operator*()
+  {
+   return *this;
+  }
+
+  reference operator=(const char32_t& value)
+  {
+   if constexpr(std::addressof(Map) != std::addressof(iso_8859_1_map_reverse)) // mapping of 128 <= x <= 255 needed
+   {
+    auto it{Map.find(value)};
+    if (it != Map.end()) {
+     s.push_back(it->second);
+     return *this;
+    }
+   }
+
+   if (value > 255)
+    throw std::invalid_argument("Bad Unicode value above 255: "s + std::to_string(static_cast<uint32_t>(value)));
+
+   s.push_back(static_cast<utf8_t>(value));
+   return *this;
+  }
+
+ private:
+  typename iso_back_insert_iterator::string_type& s;
+ };
+
+ // Facet for convert() and ISO-8859-*
+ template<typename InputIt, typename OutputIt>
+ struct ISO_8859
+ {
+  typedef utf8_t value_type;
+
+  static InputIt begin(const std::basic_string<value_type>& s)
+  {
+   return InputIt(s.cbegin());
+  }
+
+  static InputIt end(const std::basic_string<value_type>& s)
+  {
+   return InputIt(s.cend());
+  }
+
+  static OutputIt back_inserter(std::basic_string<value_type>& s)
+  {
+   return OutputIt(s);
+  }
+ };
+
+ // Facet for convert() and UTF-*
+ template<typename InputIt, typename OutputIt>
+ struct UTF
  {
-  return utf_iterator<T>{s.cbegin(), s.cend()};
+  typedef typename InputIt::value_type value_type; // OutputIt::value_type is the same
+
+  static InputIt begin(const std::basic_string<value_type>& s)
+  {
+   return InputIt{s.cbegin(), s.cend()};
+  }
+
+  static InputIt end(const std::basic_string<value_type>& s)
+  {
+   return InputIt{s.cend(), s.cend()};
+  }
+
+  static OutputIt back_inserter(std::basic_string<value_type>& s)
+  {
+   return OutputIt(s);
+  }
+ };
+
+ // Facet for convert()
+ typedef ISO_8859<iso_iterator<>, iso_back_insert_iterator<>> ISO_8859_1;
+ typedef ISO_8859<iso_iterator<iso_8859_15_map>, iso_back_insert_iterator<iso_8859_15_map_reverse>> ISO_8859_15;
+ 
+ typedef UTF<utf_iterator<utf8_t>, utf_back_insert_iterator<utf8_t>> UTF_8;
+ typedef UTF<utf_iterator<char16_t>, utf_back_insert_iterator<char16_t>> UTF_16;
+ typedef UTF<utf_iterator<char32_t>, utf_back_insert_iterator<char32_t>> UTF_32;
+
+ // From and To are facets
+ template<typename From, typename To>
+ std::basic_string<typename To::value_type> convert(const std::basic_string<typename From::value_type>& s)
+ {
+  std::basic_string<typename To::value_type> result;
+
+  std::copy(From::begin(s), From::end(s), To::back_inserter(result));
+
+  return result;
  }
 
+ // Helper to get correct Facet from char type, e.g. Encoding<typename decltype(s)::value_type>::Facet
  template<typename T>
- utf_iterator<T> utf_end(const std::basic_string<T>& s)
+ struct Encoding
  {
-  return utf_iterator<T>{s.cend(), s.cend()};
- }
+ };
 
-} // namespace
+ template<>
+ struct Encoding<utf8_t>
+ {
+  typedef UTF_8 Facet;
+ };
 
-namespace unicode {
+ template<>
+ struct Encoding<char16_t>
+ {
+  typedef UTF_16 Facet;
+ };
 
- using namespace detail;
+ template<>
+ struct Encoding<char32_t>
+ {
+  typedef UTF_32 Facet;
+ };
 
+ // From and To are from: utf8_t, char16_t and char32_t
  template<typename From, typename To>
- std::basic_string<To> utf_to_utf(const std::basic_string<From>& s)
+ std::basic_string<To> convert(const std::basic_string<From>& s)
  {
   std::basic_string<To> result;
 
-  std::copy(utf_begin<From>(s), utf_end<From>(s), utf_back_inserter<To>(result));
+  std::copy(Encoding<From>::Facet::begin(s), Encoding<From>::Facet::end(s), Encoding<To>::Facet::back_inserter(result));
 
   return result;
  }
@@ -343,7 +528,7 @@ namespace unicode {
  bool is_valid_utf(const std::basic_string<T>& s)
  {
   try {
-   std::for_each(utf_begin<T>(s), utf_end<T>(s), [](const T& c){});
+   std::for_each(Encoding<T>::Facet::begin(s), Encoding<T>::Facet::end(s), [](const T& c){});
   } catch(...) {
    return false;
   }
diff --git a/src/test-unicode.cpp b/src/test-unicode.cpp
index 3d67124..e1aa23d 100644
--- a/src/test-unicode.cpp
+++ b/src/test-unicode.cpp
@@ -96,13 +96,18 @@ void test_utf_to_utf(std::tuple<Ts...>& t)
  typedef typename std::tuple_element<i,typename std::remove_reference<decltype(t)>::type>::type From;
  typedef typename std::tuple_element<j,typename std::remove_reference<decltype(t)>::type>::type To;
 
- // test
- To result { unicode::utf_to_utf<typename From::value_type, typename To::value_type>(std::get<i>(t)) };
+ // test base type interface
+ To result { unicode::convert<typename From::value_type, typename To::value_type>(std::get<i>(t)) };
 
- BOOST_CHECK_MESSAGE(std::get<j>(t) == result, "From " << typeid(From).name() << "(" << i << ", " << std::get<i>(t) << ") to " << typeid(To).name() << "(" << j << ", " << std::get<j>(t) << "), got " << result);
+ BOOST_CHECK_MESSAGE(std::get<j>(t) == result, "Base: From " << typeid(From).name() << "(" << i << ", " << std::get<i>(t) << ") to " << typeid(To).name() << "(" << j << ", " << std::get<j>(t) << "), got " << result);
 
  //std::cout << std::to_string(std::tuple_size<typename std::remove_reference<decltype(t)>::type>::value) << "," << std::to_string(i) << "," << std::to_string(j) << std::endl;
+ 
+ // test facet interface
+ result = unicode::convert<typename unicode::Encoding<typename From::value_type>::Facet, typename unicode::Encoding<typename To::value_type>::Facet>(std::get<i>(t));
 
+ BOOST_CHECK_MESSAGE(std::get<j>(t) == result, "Facet: From " << typeid(From).name() << "(" << i << ", " << std::get<i>(t) << ") to " << typeid(To).name() << "(" << j << ", " << std::get<j>(t) << "), got " << result);
+ 
  // iterate over other combinations
  if constexpr (i + 1 < std::tuple_size<typename std::remove_reference<decltype(t)>::type>::value)
   test_utf_to_utf<i + 1, j>(t);
@@ -147,9 +152,18 @@ void test_utf_to_utf_failure(std::basic_string<From>& s)
 {
  typedef typename std::tuple_element<index, Collection>::type::value_type To;
 
+ // via base type
+ try {
+  (void) unicode::convert<From,To>(s);
+  BOOST_ERROR("Base: Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name());
+ } catch (...) {
+  // OK
+ };
+
+ // via facet
  try {
-  unicode::utf_to_utf<From,To>(s);
-  BOOST_ERROR("Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name());
+  (void) unicode::convert<typename unicode::Encoding<From>::Facet,typename unicode::Encoding<To>::Facet>(s);
+  BOOST_ERROR("Facet: Expected exception at index: " << index << ", " << typeid(From).name() << " -> " << typeid(To).name());
  } catch (...) {
   // OK
  };
@@ -236,14 +250,35 @@ void test_random(random_context& rc, size_t length)
 
  From r {generate_random<From>(rc, length)};
 
+ // base type interface
  try {
-  To result{unicode::utf_to_utf<typename From::value_type,typename To::value_type>(r)};
+  To result{unicode::convert<typename From::value_type,typename To::value_type>(r)};
+
+  if (r.empty()) {
+   BOOST_CHECK(result.empty());
+  } else {
+   BOOST_CHECK(!result.empty());
+  }
  } catch (const std::runtime_error&) {
   // OK: this is an expected exception for utf_to_utf on bad input
  } catch (const std::invalid_argument&) {
   // OK: this is an expected exception for utf_to_utf on bad input
  }
 
+ // facet interface
+ try {
+  To result{unicode::convert<typename unicode::Encoding<typename From::value_type>::Facet,typename unicode::Encoding<typename To::value_type>::Facet>(r)};
+
+  if (r.empty()) {
+   BOOST_CHECK(result.empty());
+  } else {
+   BOOST_CHECK(!result.empty());
+  }
+ } catch (const std::runtime_error&) {
+  // OK: this is an expected exception for utf_to_utf on bad input
+ } catch (const std::invalid_argument&) {
+  // OK: this is an expected exception for utf_to_utf on bad input
+ }
  //std::cerr << "DEBUG: " << typeid(From).name() << std::endl;
  //std::cerr << " DEBUG2: " << typeid(To).name() << std::endl;
 
@@ -255,8 +290,9 @@ void test_random(random_context& rc, size_t length)
 BOOST_AUTO_TEST_CASE_TEMPLATE(random_sequences, T, types_collection_type)
 {
  random_context rc;
+ int i{};
 
- // run for 1s (debug) 10s (release)
+ // run for 1s (debug) 10s (release) = total time for all random_sequences types!
 #ifdef _DEBUG
  const auto timeout{1.0s};
 #else
@@ -267,7 +303,29 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(random_sequences, T, types_collection_type)
 
  while (!(std::chrono::steady_clock::now() > timeout_stamp)) {
   test_random<T,types_collection_type>(rc, rc.sequence_length(rc.gen));
+  i++;
  }
+
+ BOOST_CHECK_MESSAGE(i > 1, "Not enough iterations done!");
+}
+
+// Test ISO and UTF encodings
+BOOST_AUTO_TEST_CASE(convert)
+{
+ BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>({})}) == std::string{});
+ BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>("abc")}) == std::string{"abc"});
+ BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>("äöü")}) == std::string{"äöü"});
+ BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_1,unicode::ISO_8859_1>("\xa4")}) == std::string{"\xa4"}); // €
+ 
+ BOOST_CHECK((std::string{unicode::convert<unicode::ISO_8859_15,unicode::ISO_8859_15>("\xa4")}) == std::string{"\xa4"}); // €
+ 
+ BOOST_CHECK_THROW(((void)std::string{unicode::convert<unicode::ISO_8859_15,unicode::ISO_8859_1>("\xa4")}), std::invalid_argument); // € not available in ISO-8859-1
+ 
+ BOOST_CHECK((unicode::convert<unicode::UTF_8,unicode::UTF_16>("abc")) == std::u16string{u"abc"});
+ BOOST_CHECK((unicode::convert<unicode::UTF_32,unicode::UTF_16>(U"abc")) == std::u16string{u"abc"});
+ 
+ BOOST_CHECK((unicode::convert<utf8_t,char16_t>("abc")) == std::u16string{u"abc"});
+ BOOST_CHECK((unicode::convert<char32_t,char16_t>(U"abc")) == std::u16string{u"abc"});
 }
 
 // TODO:
diff --git a/src/validate.cpp b/src/validate.cpp
new file mode 100644
index 0000000..8927fe4
--- /dev/null
+++ b/src/validate.cpp
@@ -0,0 +1,4 @@
+int main(int argc, char* argv[])
+{
+ return 0;
+}