read hex byte into control for unicode (#29)

- possibly fixes #26

rtftxt/rtftxt.go

@@ -126,7 +126,16 @@ func tokenizeControl(r peekingReader.Reader) (string, int, error) {
  case b == '\'' && buf.Len() == 0:
  isHex = true
  buf.WriteByte(b)
- r.ReadByte()
+ r.ReadByte() // consume valid character
+ // read 2 bytes for hex
+ for i := 0; i < 2; i++ {
+ b, err = r.ReadByte() // consume valid digit
+ if err != nil {
+ return "", -1, err
+ }
+ buf.WriteByte(b)
+ }
+ return buf.String(), -1, nil
  case b >= '0' && b <= '9' || b == '-':
  if numStart == -1 {
  numStart = buf.Len()

rtftxt/rtftxt_test.go

@@ -8,6 +8,7 @@ import (
 
 const txt = `Of course, we frequently hear about larger brands pushing out a ton of amazing content, and they're often used as examples of how to do content right. `
 const red = `Restore The Selling Balance. Ad Technology doesn't have to be faceless. Our platform is designed to connect media companies directly to advertisers.`
+const unicode = `Beställer! Bulkpackning `
 
 func TestToStr(t *testing.T) {
  s, err := rtftxt.ToStr(`../test/np.new.rtf`)
@@ -29,4 +30,12 @@ func TestToStr(t *testing.T) {
  if err == nil {
  t.Error("Nonexisting file does not throw error")
  }
+
+ s, err = rtftxt.ToStr(`../test/infotext.rtf`)
+ if err != nil {
+ t.Error(err)
+ }
+ if s != unicode {
+ t.Error("doesn't match", s)
+ }
 }

test/infotext.rtf

@@ -0,0 +1,5 @@
+{\rtf1\ansi\ansicpg1252\deff0\deflang1053{\fonttbl{\f0\fnil\fcharset0 Arial;}}
+\viewkind4\uc1\pard\fs48 Best\'e4ller!
+\par Bulkpackning
+\par 
+\par }