[hfst] 01/02: Imported Upstream version 3.8.3~r4409
Tino Didriksen
tinodidriksen-guest at moszumanska.debian.org
Mon Aug 3 06:58:59 UTC 2015
This is an automated email from the git hooks/post-receive script.
tinodidriksen-guest pushed a commit to branch master
in repository hfst.
commit 6ea981e50be935279740903859686db96a0a0f38
Author: Tino Didriksen <mail at tinodidriksen.com>
Date: Mon Aug 3 06:57:08 2015 +0000
Imported Upstream version 3.8.3~r4409
---
ChangeLog | 1044 ++++++++++++++
ChangeLog.old | 424 ++++++
NEWS | 10 +-
back-ends/foma/Makefile.am | 2 +-
back-ends/foma/apply.c | 3 +-
back-ends/foma/constructions.c | 2 +-
back-ends/foma/determinize.c | 12 +-
back-ends/foma/fomalib.h | 18 +-
back-ends/foma/fomalibconf.h | 10 +
back-ends/foma/iface.c | 6 +-
back-ends/foma/io.c | 8 +-
back-ends/foma/lex.cmatrix.c | 8 +-
back-ends/foma/lex.lexc.c | 2 +-
back-ends/foma/lex.yy.c | 15 +-
back-ends/foma/mem.c | 8 +-
back-ends/foma/minimize.c | 8 +-
back-ends/foma/sigma.c | 17 +-
back-ends/foma/spelling.c | 6 +-
back-ends/foma/structures.c | 26 +-
back-ends/openfst/src/include/fst/rmfinalepsilon.h | 4 +-
.../openfst/src/include/fst/symbol-table-ops.h | 4 +-
back-ends/openfst/src/include/fst/synchronize.h | 4 +-
back-ends/openfstwin/src/include/fst/compat.h | 18 +-
back-ends/openfstwin/src/include/fst/flags.h | 8 +-
back-ends/openfstwin/src/include/fst/properties.h | 2 +-
configure.ac | 45 +-
libhfst/src/FormatSpecifiers.h | 2 +-
libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc | 6 +-
libhfst/src/HarmonizeUnknownAndIdentitySymbols.h | 19 +-
libhfst/src/HfstApply.cc | 53 +-
libhfst/src/HfstDataTypes.cc | 6 +
libhfst/src/HfstDataTypes.h | 1 +
libhfst/src/HfstEpsilonHandler.cc | 4 +-
libhfst/src/HfstExceptionDefs.h | 4 +-
libhfst/src/HfstFlagDiacritics.cc | 18 +-
libhfst/src/HfstFlagDiacritics.h | 26 +-
libhfst/src/HfstInputStream.cc | 124 +-
libhfst/src/HfstInputStream.h | 26 +-
libhfst/src/HfstLookupFlagDiacritics.cc | 54 +-
libhfst/src/HfstOutputStream.cc | 70 +-
libhfst/src/HfstOutputStream.h | 29 +-
libhfst/src/HfstRules.cc | 13 +-
libhfst/src/HfstSymbolDefs.cc | 19 +-
libhfst/src/HfstSymbolDefs.h | 41 +-
libhfst/src/HfstTokenizer.cc | 10 +-
libhfst/src/HfstTokenizer.h | 30 +-
libhfst/src/HfstTransducer.cc | 672 ++++++++-
libhfst/src/HfstTransducer.h | 401 +++---
libhfst/src/HfstXeroxRules.cc | 10 +-
libhfst/src/Makefile.am | 10 +-
libhfst/src/hfst_apply_schemas.h | 12 +
libhfst/src/hfstdll.h | 12 +
.../src/implementations/ConvertFomaTransducer.cc | 4 +-
libhfst/src/implementations/ConvertOlTransducer.cc | 5 +-
.../src/implementations/ConvertTransducerFormat.cc | 6 +
.../src/implementations/ConvertTransducerFormat.h | 15 +-
.../ConvertTropicalWeightTransducer.cc | 14 +-
.../src/implementations/ConvertXfsmTransducer.cc | 252 ++++
libhfst/src/implementations/FomaTransducer.cc | 10 +-
libhfst/src/implementations/FomaTransducer.h | 7 +-
libhfst/src/implementations/HfstTransition.h | 100 +-
libhfst/src/implementations/HfstTransitionGraph.h | 328 +++--
.../HfstTropicalTransducerTransitionData.h | 56 +-
libhfst/src/implementations/LogWeightTransducer.cc | 73 +-
libhfst/src/implementations/LogWeightTransducer.h | 13 +-
libhfst/src/implementations/Makefile.am | 27 +-
.../implementations/TropicalWeightTransducer.cc | 117 +-
.../src/implementations/TropicalWeightTransducer.h | 9 +-
libhfst/src/implementations/XfsmTransducer.cc | 555 ++++++++
libhfst/src/implementations/XfsmTransducer.h | 164 +++
.../compose_intersect/ComposeIntersectFst.cc | 12 +-
.../compose_intersect/ComposeIntersectFst.h | 16 +-
.../compose_intersect/ComposeIntersectLexicon.cc | 90 +-
.../compose_intersect/ComposeIntersectLexicon.h | 18 +-
.../compose_intersect/ComposeIntersectRulePair.cc | 14 +-
.../compose_intersect/ComposeIntersectRulePair.h | 2 +-
.../compose_intersect/ComposeIntersectUtilities.cc | 8 +-
.../compose_intersect/ComposeIntersectUtilities.h | 6 +-
.../implementations/optimized-lookup/convert.cc | 2 +-
.../src/implementations/optimized-lookup/pmatch.cc | 46 +-
.../src/implementations/optimized-lookup/pmatch.h | 6 +-
.../implementations/optimized-lookup/transducer.cc | 3 +-
.../implementations/optimized-lookup/transducer.h | 23 +-
libhfst/src/parsers/LexcCompiler.cc | 76 +-
libhfst/src/parsers/pmatch_lex.ll | 16 +-
libhfst/src/parsers/pmatch_parse.yy | 49 +-
libhfst/src/parsers/pmatch_utils.cc | 51 +-
libhfst/src/parsers/pmatch_utils.h | 19 +
libhfst/src/parsers/xre_lex.ll | 5 +-
libhfst/src/parsers/xre_parse.yy | 20 +-
libhfst/src/parsers/xre_utils.cc | 6 +-
scripts/README_eight_tools.txt | 91 ++
scripts/README_xfst.txt | 59 +
scripts/copy-for-windows.sh | 341 +++++
scripts/generate-python-bindings.bat | 4 +
scripts/generate-static-binaries.sh | 67 +
scripts/hfst-twolc-bin | 5 +
scripts/inttypes.h | 305 ++++
scripts/libhfst_win.i | 222 +++
scripts/make-foma.bat | 5 +
scripts/make-hfst-lexc.bat | 87 ++
scripts/make-hfst-proc.bat | 97 ++
scripts/make-hfst-tool.bat | 87 ++
scripts/make-hfst-tools.bat | 1 +
scripts/make-hfst-xfst.bat | 92 ++
scripts/make-htwolcpre1.bat | 95 ++
scripts/make-htwolcpre2.bat | 89 ++
scripts/make-htwolcpre3.bat | 103 ++
scripts/make-implementations.bat | 22 +
scripts/make-libhfst.bat | 59 +
scripts/make-openfstwin.bat | 12 +
scripts/make-parsers.bat | 1 +
scripts/make-python-bindings.bat | 92 ++
scripts/package-static-binaries.sh | 31 +
scripts/stdint.h | 247 ++++
scripts/test-hfst-tools.bat | 31 +
scripts/test-libhfst.bat | 2 +
scripts/test-openfstwin.bat | 2 +
scripts/test_libhfst_win.py | 56 +
scripts/windows_tests/test.lexc | 4 +
scripts/windows_tests/test.pmatch | 5 +
scripts/windows_tests/test.twolc | 4 +
scripts/windows_tests/test.xfst | 5 +
scripts/windows_tests/test_lexc_result.txt | 4 +
scripts/windows_tests/test_pmatch_result.txt | 1 +
scripts/windows_tests/test_twolc_result.txt | 3 +
scripts/windows_tests/test_xfst_result.txt | 3 +
swig/libhfst.i | 16 +-
swig/setup.py | 2 +-
test/tools/proc-functionality.sh | 2 +-
tools/src/HfstStrings2FstTokenizer.cc | 42 +-
tools/src/HfstStrings2FstTokenizer.h | 8 +-
tools/src/Makefile.am | 4 +-
tools/src/hfst-commandline.cc | 31 +-
tools/src/hfst-commandline.h | 6 +
tools/src/hfst-compare.cc | 17 +-
tools/src/hfst-compose-intersect.cc | 63 +-
tools/src/hfst-compose.cc | 46 +-
tools/src/hfst-concatenate.cc | 1 +
tools/src/hfst-conjunct.cc | 1 +
tools/src/hfst-disjunct.cc | 1 +
tools/src/hfst-fst2fst.cc | 34 +-
tools/src/hfst-fst2strings.cc | 58 +-
tools/src/hfst-fst2txt.cc | 85 +-
tools/src/hfst-getopt.cc | 182 +++
tools/src/hfst-getopt.h | 18 +
tools/src/hfst-head.cc | 1 +
tools/src/hfst-info.cc | 10 +
tools/src/hfst-lexc-compiler.cc | 15 +-
tools/src/hfst-lookup.cc | 271 +++-
tools/src/hfst-minimize.cc | 26 +-
tools/src/hfst-optimized-lookup.cc | 254 +++-
tools/src/hfst-optimized-lookup.h | 1503 ++++++++++----------
tools/src/hfst-pair-test.cc | 272 +++-
tools/src/hfst-pmatch.cc | 50 +-
tools/src/hfst-pmatch2fst.cc | 199 ++-
tools/src/hfst-proc/alphabet.cc | 2 +-
tools/src/hfst-proc/hfst-proc.cc | 46 +-
tools/src/hfst-proc/tokenizer.cc | 13 +-
tools/src/hfst-proc2.cc | 190 ++-
tools/src/hfst-reweight.cc | 60 +-
tools/src/hfst-split.cc | 2 +
tools/src/hfst-string-conversions.cc | 133 +-
tools/src/hfst-string-conversions.h | 32 +-
tools/src/hfst-strings2fst.cc | 14 +-
tools/src/hfst-subtract.cc | 1 +
tools/src/hfst-summarize.cc | 7 +-
tools/src/hfst-tail.cc | 1 +
tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc | 70 +-
tools/src/hfst-twolc/src/alphabet_src/Alphabet.h | 4 +
.../hfst-twolc/src/commandline_src/CommandLine.cc | 6 +-
.../hfst-twolc/src/commandline_src/CommandLine.h | 9 +-
tools/src/hfst-twolc/src/hfst-twolc.bat | 2 +-
tools/src/hfst-twolc/src/htwolcpre1.yy | 14 +-
tools/src/hfst-twolc/src/htwolcpre2.yy | 26 +-
tools/src/hfst-twolc/src/htwolcpre3.yy | 12 +-
.../src/rule_src/ConflictResolvingLeftArrowRule.cc | 2 +-
.../src/rule_src/OtherSymbolTransducer.cc | 30 +-
tools/src/hfst-twolc/src/rule_src/Rule.cc | 2 +-
tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc | 18 +-
tools/src/hfst-twolc/src/scanner1.ll | 8 +-
.../src/string_src/string_manipulation.cc | 2 +-
.../src/variable_src/ConstContainerIterator.h | 4 +-
.../src/variable_src/MixedConstContainerIterator.h | 4 +-
.../src/variable_src/RuleSymbolVector.cc | 4 +-
.../hfst-twolc/src/variable_src/RuleVariables.cc | 4 +-
.../src/variable_src/RuleVariablesConstIterator.cc | 2 +-
.../src/variable_src/VariableValueIterator.h | 6 +-
tools/src/hfst-txt2fst.cc | 55 +-
tools/src/parsers/Makefile.am | 4 +-
tools/src/parsers/XfstCompiler.cc | 220 ++-
tools/src/parsers/XfstCompiler.h | 7 +-
tools/src/parsers/hfst-xfst.cc | 96 +-
tools/src/parsers/test/Makefile.am | 2 +
tools/src/parsers/test/substitute_symbol_7.att | 8 +
tools/src/parsers/test/substitute_symbol_7.xfst | 7 +
tools/src/parsers/test/substitute_symbol_8.att | 7 +
tools/src/parsers/test/substitute_symbol_8.xfst | 7 +
tools/src/parsers/test/test.sh | 109 +-
tools/src/parsers/xfst-lexer.ll | 6 +-
tools/src/parsers/xfst-parser.yy | 50 +-
tools/src/parsers/xfst-utils.h | 6 +
.../{help_message.cc => xfst_help_message.cc} | 3 +
tools/src/parsers/xfst_help_message.h | 61 +
204 files changed, 10188 insertions(+), 2371 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 50487c4..79d4943 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,1047 @@
+2015-06-11 10:24 eaxelson
+
+ * scripts/README_eight_tools.txt, scripts/copy-for-windows.sh:
+ Updated windows scripts.
+
+2015-06-11 10:24 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Now a warning message gets
+ printed if stack contains transducers whose types differ (can
+ happen if lookup-optimize is called before pushing another
+ transducer onto stack).
+
+2015-06-10 10:49 eaxelson
+
+ * scripts/README_eight_tools.txt: Added a windows README file for
+ the package that contains eight command line tools.
+
+2015-06-10 09:54 eaxelson
+
+ * scripts/README_xfst.txt: Added a Windows README file for
+ hfst-xfst.
+
+2015-06-09 13:09 eaxelson
+
+ * back-ends/openfst/src/include/fst/rmfinalepsilon.h,
+ back-ends/openfst/src/include/fst/symbol-table-ops.h,
+ back-ends/openfst/src/include/fst/synchronize.h,
+ scripts/copy-for-windows.sh, scripts/test-hfst-tools.bat,
+ scripts/windows_tests, scripts/windows_tests/test.lexc,
+ scripts/windows_tests/test.pmatch,
+ scripts/windows_tests/test.twolc,
+ scripts/windows_tests/test.xfst,
+ scripts/windows_tests/test_lexc_result.txt,
+ scripts/windows_tests/test_pmatch_result.txt,
+ scripts/windows_tests/test_twolc_result.txt,
+ scripts/windows_tests/test_xfst_result.txt,
+ tools/src/hfst-proc/hfst-proc.cc,
+ tools/src/hfst-string-conversions.cc,
+ tools/src/hfst-string-conversions.h,
+ tools/src/hfst-twolc/src/hfst-twolc.bat: Added tests for windows.
+ Made small fixes to hfst-proc and hfst-twolc for windows.
+
+2015-06-08 12:31 eaxelson
+
+ * tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc,
+ tools/src/hfst-twolc/src/commandline_src/CommandLine.cc,
+ tools/src/hfst-twolc/src/htwolcpre1.yy,
+ tools/src/hfst-twolc/src/htwolcpre2.yy,
+ tools/src/hfst-twolc/src/htwolcpre3.yy,
+ tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc,
+ tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc,
+ tools/src/hfst-twolc/src/rule_src/Rule.cc,
+ tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc,
+ tools/src/hfst-twolc/src/scanner1.ll,
+ tools/src/hfst-twolc/src/string_src/string_manipulation.cc,
+ tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h,
+ tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h,
+ tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc,
+ tools/src/hfst-twolc/src/variable_src/RuleVariables.cc,
+ tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc,
+ tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h:
+ Replaced alternative tokens for logical operators with the
+ standard ones.
+
+2015-06-08 12:22 eaxelson
+
+ * tools/src/hfst-twolc/src/commandline_src/CommandLine.h: Checking
+ for existence of config.h and getopt.h.
+
+2015-06-08 12:19 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-hfst-tools.bat,
+ scripts/make-htwolcpre2.bat: Updated scripts.
+
+2015-06-08 10:02 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-htwolcpre1.bat,
+ scripts/make-htwolcpre2.bat, scripts/make-htwolcpre3.bat: Added
+ make scripts for twolc.
+
+2015-06-08 08:26 eaxelson
+
+ * scripts/copy-for-windows.sh: Added twolc to windows compilation
+ scripts.
+
+2015-06-05 13:46 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-hfst-proc.bat,
+ scripts/make-hfst-xfst.bat: Updated windows scripts.
+
+2015-06-05 13:43 eaxelson
+
+ * tools/src/Makefile.am, tools/src/hfst-lookup.cc,
+ tools/src/hfst-optimized-lookup.cc, tools/src/hfst-pmatch.cc,
+ tools/src/hfst-pmatch2fst.cc,
+ tools/src/hfst-string-conversions.cc,
+ tools/src/hfst-string-conversions.h: Fixing again reading and
+ writing to/from console on windows.
+
+2015-06-04 12:58 eaxelson
+
+ * libhfst/src/parsers/pmatch_utils.h: Using hex values for unicode
+ characters when compiling with msvc to get rid of warnings (and
+ possible segfaults).
+
+2015-06-04 08:24 jezral
+
+ * test/tools/proc-functionality.sh,
+ tools/src/hfst-proc/tokenizer.cc: Terminator needs reading, and
+ yes there are partial reads
+
+2015-06-03 14:34 jezral
+
+ * tools/src/hfst-proc/tokenizer.cc: std::string all the way
+
+2015-06-03 10:29 eaxelson
+
+ * tools/src/hfst-proc/alphabet.cc,
+ tools/src/hfst-proc/hfst-proc.cc,
+ tools/src/hfst-proc/tokenizer.cc: Added missing HAVE_CONFIG_H
+ ifdef. Added globals-common.h header as some compilers complain
+ about missing extern variables. Now creating variable size array
+ 'next_u8' dynamically with operator 'new'.
+
+2015-06-03 10:04 eaxelson
+
+ * tools/src/parsers/Makefile.am, tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/help_message.cc,
+ tools/src/parsers/xfst_help_message.cc,
+ tools/src/parsers/xfst_help_message.h: Renamed help_message.cc
+ into xfst_help_message.cc and added a header file
+ xfst_help_message.h. Now XfstCompiler includes only the header
+ file.
+
+2015-06-03 07:53 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Fixed a typo and added
+ explicit braces to an else statement.
+
+2015-06-02 12:54 eaxelson
+
+ * tools/src/hfst-getopt.cc, tools/src/hfst-lookup.cc,
+ tools/src/hfst-optimized-lookup.cc,
+ tools/src/hfst-string-conversions.cc,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/hfst-xfst.cc, tools/src/parsers/test/test.sh:
+ Now option --pipe-mode takes an optional argument { input,
+ output, both }, 'both' being the default.
+
+2015-05-29 12:25 mpsilfve
+
+ * libhfst/src/Makefile.am: Install hfstdll.h
+
+2015-05-29 11:24 eaxelson
+
+ * tools/src/Makefile.am, tools/src/hfst-lookup.cc,
+ tools/src/hfst-optimized-lookup.cc,
+ tools/src/hfst-string-conversions.cc,
+ tools/src/hfst-string-conversions.h,
+ tools/src/parsers/hfst-xfst.cc: Now hfst-lookup and
+ hfst-optimized-lookup read from console and print to console by
+ default on windows.
+
+2015-05-28 12:21 eaxelson
+
+ * tools/src/hfst-optimized-lookup.cc: Commented out one debug
+ print...
+
+2015-05-28 12:20 eaxelson
+
+ * tools/src/hfst-optimized-lookup.cc,
+ tools/src/hfst-string-conversions.cc,
+ tools/src/hfst-string-conversions.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/hfst-xfst.cc: More improvements to utf-8
+ handling on windows.
+
+2015-05-27 13:09 eaxelson
+
+ * tools/src/Makefile.am, tools/src/hfst-optimized-lookup.cc,
+ tools/src/hfst-string-conversions.cc,
+ tools/src/hfst-string-conversions.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/XfstCompiler.h: Small modifications for better
+ utf-8 support on windows.
+
+2015-05-26 12:06 eaxelson
+
+ * scripts/copy-for-windows.sh, tools/src/hfst-lookup.cc,
+ tools/src/hfst-optimized-lookup.cc,
+ tools/src/hfst-optimized-lookup.h, tools/src/hfst-pmatch.cc,
+ tools/src/hfst-pmatch2fst.cc: Small modifications for some
+ command line tools for windows.
+
+2015-05-25 13:08 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-hfst-lexc.bat,
+ scripts/make-hfst-tool.bat, tools/src/hfst-compare.cc,
+ tools/src/hfst-strings2fst.cc, tools/src/hfst-txt2fst.cc,
+ tools/src/parsers/test/test.sh: Added tool compilation scripts
+ for windows. Argument handling on windows is also supported for
+ some tools needed for hfst-xfst testing.
+
+2015-05-25 12:56 eaxelson
+
+ * tools/src/hfst-getopt.cc: Fixed a bug in argument handling.
+
+2015-05-22 11:14 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-hfst-xfst.bat,
+ tools/src/hfst-lexc-compiler.cc, tools/src/parsers/hfst-xfst.cc:
+ Edited help messages and warnings of hfst-xfst and hfst-lexc on
+ windows about character encodings.
+
+2015-05-21 13:06 eaxelson
+
+ * tools/src/hfst-getopt.cc, tools/src/hfst-getopt.h,
+ tools/src/hfst-lexc-compiler.cc, tools/src/parsers/hfst-xfst.cc:
+ Moved getopt implementation to a separate file.
+
+2015-05-20 14:01 eaxelson
+
+ * scripts/copy-for-windows.sh, tools/src/parsers/hfst-xfst.cc:
+ Added an implementation for long options for windows.
+
+2015-05-19 11:50 eaxelson
+
+ * back-ends/foma/lex.cmatrix.c, back-ends/foma/sigma.c,
+ back-ends/foma/spelling.c, back-ends/foma/structures.c,
+ scripts/copy-for-windows.sh, tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/hfst-xfst.cc: Fixed some issues noticed during
+ msvc compilation. Renamed function 'min' to 'min_' to avoid
+ collision with a macro with the same name. Changed arguments of
+ comparison function given to qsort to (const void *, const void
+ *). Added support for commandline arguments of type
+ '--option=argument' to hfst-xfst for windows. Also changed
+ hfst-xfst's option --print-to-console (default false) into
+ --no-console (default false) on windows.
+
+2015-05-18 13:41 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-hfst-xfst.bat,
+ tools/src/getopt_long.cc, tools/src/getopt_long.h: Updated
+ windows script files.
+
+2015-05-18 13:40 eaxelson
+
+ * tools/src/parsers/hfst-xfst.cc: Deleted getopt_long cc and h
+ files. Processing command line arguments manually in hfst-xfst.cc
+ on windows.
+
+2015-05-18 13:39 eaxelson
+
+ * libhfst/src/implementations/optimized-lookup/transducer.h,
+ tools/src/hfst-commandline.cc: Added some ifdef _MSC_VER
+ directives.
+
+2015-05-18 10:36 eaxelson
+
+ * libhfst/src/implementations/optimized-lookup/transducer.h:
+ Applied patch for musl libc compilation.
+
+2015-05-11 12:04 eaxelson
+
+ * libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc,
+ libhfst/src/HarmonizeUnknownAndIdentitySymbols.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/implementations/optimized-lookup/transducer.h,
+ scripts/make-hfst-xfst.bat, tools/src/hfst-commandline.cc,
+ tools/src/parsers/XfstCompiler.cc: Fixed some bugs noticed when
+ compiling on windows.
+
+2015-05-08 20:21 hardwick
+
+ * tools/src/hfst-proc2.cc: Blank line between outputs in --finnpos
+ mode
+
+2015-05-08 14:05 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/libhfst_win.i,
+ scripts/make-hfst-xfst.bat,
+ tools/src/HfstStrings2FstTokenizer.cc, tools/src/getopt_long.cc,
+ tools/src/getopt_long.h, tools/src/hfst-commandline.cc,
+ tools/src/hfst-commandline.h, tools/src/parsers/hfst-xfst.cc,
+ tools/src/parsers/xfst-lexer.ll, tools/src/parsers/xfst-utils.h:
+ Updated windows scripts. Added ifdefs for msvc compilation.
+
+2015-05-05 12:15 eaxelson
+
+ * scripts/libhfst_win.i, scripts/make-python-bindings.bat,
+ scripts/test_libhfst_win.py: Updated swig scripts for windows.
+
+2015-04-29 12:32 hardwick
+
+ * tools/src/hfst-proc2.cc: Add FinnPos mode and reorganise
+
+2015-04-27 13:34 eaxelson
+
+ * back-ends/openfstwin/src/include/fst/flags.h,
+ scripts/copy-for-windows.sh, scripts/make-foma.bat,
+ scripts/make-libhfst.bat, scripts/make-openfstwin.bat,
+ scripts/make-python-bindings.bat, scripts/test-libhfst.bat,
+ scripts/test-openfstwin.bat, scripts/test_libhfst_win.py: Updated
+ and added scripts for compiling python bindings on windows.
+
+2015-04-23 11:30 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.cc,
+ tools/src/hfst-pmatch2fst.cc: Fix harmonization issue and speed
+ up the common case of only on top-level matcher
+
+2015-04-23 06:24 moshagen
+
+ * tools/src/hfst-reweight.cc: Documented the tsv file format, added
+ some linebreaks to make the helpt text easier to read.
+
+2015-04-21 09:22 moshagen
+
+ * tools/src/hfst-reweight.cc: Always skip comment lines. Added
+ verbose output for TSV file reweighting.
+
+2015-04-21 07:21 moshagen
+
+ * tools/src/hfst-reweight.cc: Fixed bug #293 by initialising the
+ line variable before using it.
+
+2015-04-20 13:52 moshagen
+
+ * tools/src/hfst-reweight.cc: Whitespace only.
+
+2015-04-17 18:54 eaxelson
+
+ * libhfst/src/hfstdll.h: Added missing file 'hfstdll.h'.
+
+2015-04-17 12:43 eaxelson
+
+ * scripts/copy-for-windows.sh,
+ scripts/generate-python-bindings.bat, scripts/libhfst_win.i,
+ scripts/make-libhfst.bat, scripts/make-openfstwin.bat,
+ scripts/test-libhfst.bat, scripts/test-openfstwin.bat: Edited and
+ added scripts for windows compilation.
+
+2015-04-17 12:10 eaxelson
+
+ * libhfst/src/HarmonizeUnknownAndIdentitySymbols.h,
+ libhfst/src/HfstExceptionDefs.h,
+ libhfst/src/HfstFlagDiacritics.h, libhfst/src/HfstInputStream.h,
+ libhfst/src/HfstOutputStream.h, libhfst/src/HfstSymbolDefs.h,
+ libhfst/src/HfstTokenizer.h, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/HfstTransition.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/implementations/HfstTropicalTransducerTransitionData.h:
+ Added HFSTDLL's for some functions for windows compilation.
+
+2015-04-15 11:48 eaxelson
+
+ * tools/src/hfst-reweight.cc: Added option --arcs-only to
+ hfst-reweight. Now weights of all arcs and end states are
+ modified unless --arcs-only or --end-states-only is used. Also
+ fixed '--end-state-only' as '--end-states-only'.
+
+2015-04-13 10:26 eaxelson
+
+ * libhfst/src/implementations/ConvertOlTransducer.cc,
+ libhfst/src/implementations/FomaTransducer.cc,
+ libhfst/src/implementations/LogWeightTransducer.cc,
+ libhfst/src/implementations/TropicalWeightTransducer.cc: Added
+ some explicit type conversions.
+
+2015-04-09 13:28 moshagen
+
+ * tools/src/hfst-pair-test.cc: Changed the short form of the xerox
+ mode from -x to -X (upper case), to be more consistent with the
+ other options, and to match the test on line 195.
+
+2015-04-08 13:12 eaxelson
+
+ * configure.ac, libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/FomaTransducer.cc,
+ libhfst/src/implementations/FomaTransducer.h,
+ libhfst/src/implementations/Makefile.am, tools/src/Makefile.am:
+ Now it is checked if foma back-end and zlib are available before
+ hfst-lexc-wrapper is generated. This also affects the way
+ HfstTransducer::read_lexc_ptr is implemented for FOMA_TYPE:
+ native HFST lexc compiler is used if lexc wrapper is not
+ generated.
+
+2015-04-08 13:09 eaxelson
+
+ * swig/libhfst.i, swig/setup.py: Updated hfst version number in the
+ python setup file. Fixed the argument type of
+ LexcCompiler::setVerbosity from bool to unsigned int.
+
+2015-04-08 13:06 eaxelson
+
+ * libhfst/src/HfstInputStream.cc,
+ libhfst/src/implementations/ConvertFomaTransducer.cc,
+ libhfst/src/implementations/ConvertTropicalWeightTransducer.cc:
+ Changed again some 'not' operators into \!.
+
+2015-04-08 13:01 eaxelson
+
+ * libhfst/src/implementations/optimized-lookup/convert.cc: Added a
+ missing // in the beginning of a comment line noticed during
+ windows compilation.
+
+2015-04-08 13:00 eaxelson
+
+ * scripts/copy-for-windows.sh,
+ scripts/generate-python-bindings.bat, scripts/make-libhfst.bat:
+ Edited windows batch scripts and added a new one for creating
+ python bindings.
+
+2015-04-08 09:48 eaxelson
+
+ * libhfst/src/parsers/xre_parse.yy,
+ libhfst/src/parsers/xre_utils.cc: Changed declaration 'class
+ yy_buffer_state' into 'struct yy_buffer_state'.
+
+2015-04-07 14:23 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Fixed declared return type
+ from 'void' into 'int' for functions hlexclex_destroy and
+ pmatcherror.
+
+2015-04-07 13:24 mpsilfve
+
+ * tools/src/hfst-pair-test.cc: Xerox twolc test support in
+ hfst-pair-test
+
+2015-04-02 14:37 mpsilfve
+
+ * libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h:
+ Tentatively fixed missing paths problem in compose-intersect
+
+2015-04-01 12:21 eaxelson
+
+ * libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h:
+ Moved definition of member 'static const HfstState START' of
+ classes ComposeIntersectFst and ComposeIntersectRulePair from
+ header to cc file.
+
+2015-04-01 11:39 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Add [].t() syntax for
+ delimiting tagged (and context-checked!) regions
+
+2015-03-31 14:47 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/make-implementations.bat,
+ scripts/make-libhfst.bat: Added new scripts for testing native
+ windows compilation.
+
+2015-03-31 14:46 eaxelson
+
+ * libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc,
+ libhfst/src/HfstEpsilonHandler.cc,
+ libhfst/src/HfstInputStream.cc, libhfst/src/HfstOutputStream.cc,
+ libhfst/src/HfstRules.cc, libhfst/src/HfstTokenizer.cc,
+ libhfst/src/HfstTransducer.cc, libhfst/src/HfstXeroxRules.cc,
+ libhfst/src/implementations/LogWeightTransducer.cc: Added missing
+ return values for some functions. Replaced ciso646 keywords for
+ logical operators with standard ones.
+
+2015-03-26 22:14 eaxelson
+
+ * scripts/make-implementations.bat, scripts/make-parsers.bat: Added
+ make scripts for windows.
+
+2015-03-26 22:13 eaxelson
+
+ * scripts/copy-for-windows.sh: Updated script.
+
+2015-03-26 21:36 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/pmatch_utils.cc,
+ tools/src/HfstStrings2FstTokenizer.cc,
+ tools/src/HfstStrings2FstTokenizer.h, tools/src/hfst-lookup.cc,
+ tools/src/hfst-pair-test.cc, tools/src/hfst-strings2fst.cc:
+ Removed unnecessary HfstUtf8.h and HfstStrings2FstTokenizer.h
+ inclusions from libhfst/src/ files. Also moved
+ HfstStrings2FstTokenizer into hfst namespace.
+
+2015-03-26 20:44 eaxelson
+
+ * back-ends/foma/fomalib.h, back-ends/foma/fomalibconf.h,
+ libhfst/src/FormatSpecifiers.h,
+ libhfst/src/implementations/ConvertTransducerFormat.h,
+ libhfst/src/implementations/FomaTransducer.h,
+ libhfst/src/implementations/LogWeightTransducer.h: Now handling
+ all stdbool definitions in foma headers.
+
+2015-03-26 15:05 eaxelson
+
+ * libhfst/src/implementations/FomaTransducer.h,
+ libhfst/src/implementations/LogWeightTransducer.h,
+ libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/TropicalWeightTransducer.h: Now
+ HAVE_CONFIG_H is checked before including config.h. Removed
+ inclusion of zlib.h that had no effect.
+
+2015-03-26 15:02 eaxelson
+
+ * back-ends/foma/Makefile.am, back-ends/foma/iface.c,
+ back-ends/foma/io.c, back-ends/foma/lex.lexc.c,
+ back-ends/foma/lex.yy.c: Changed ifdef WINDOWS into _MSC_VER. Now
+ zlib.h is included only when requested with -DZLIB.
+
+2015-03-26 12:20 eaxelson
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Using
+ dynamic arrays instead of static ones to keep cl.exe happy. Also
+ added definition for 'ssize_t' on windows.
+
+2015-03-26 12:18 eaxelson
+
+ * libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc:
+ Replaced 'and' with '&&'.
+
+2015-03-26 12:17 eaxelson
+
+ * libhfst/src/implementations/FomaTransducer.cc: Replaced 'and'
+ with '&&'. Also made a variable defining the size of an array
+ const.
+
+2015-03-25 13:34 eaxelson
+
+ * libhfst/src/implementations/ConvertOlTransducer.cc,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc,
+ libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h,
+ libhfst/src/implementations/optimized-lookup/transducer.h:
+ Replaced ciso646 aliases and,or,not with &&,||,\!. Also changed
+ sizeof(variable) into sizeof(variable_type) in ol transducer
+ header because cl.exe complained about that for some reason.
+
+2015-03-25 12:47 eaxelson
+
+ * libhfst/src/parsers/xre_utils.cc: Fixed a bug where operator '=='
+ was used instead of '=' when setting the value of last element in
+ a char array.
+
+2015-03-23 14:58 eaxelson
+
+ * scripts/copy-for-windows.sh: Added missing files.
+
+2015-03-20 15:11 eaxelson
+
+ * scripts/copy-for-windows.sh, scripts/inttypes.h,
+ scripts/make-foma.bat, scripts/make-openfstwin.bat,
+ scripts/stdint.h: Added scripts and headers for windows
+ compilation.
+
+2015-03-20 15:10 eaxelson
+
+ * libhfst/src/HfstFlagDiacritics.cc,
+ libhfst/src/HfstLookupFlagDiacritics.cc,
+ libhfst/src/HfstSymbolDefs.cc,
+ libhfst/src/implementations/TropicalWeightTransducer.cc,
+ libhfst/src/implementations/TropicalWeightTransducer.h: Changed
+ operators {not,and,or} into {\!,&&,||} because cl.exe complains
+ about them on windows.
+
+2015-03-20 14:20 eaxelson
+
+ * scripts/copy-for-windows.sh: Added a script for windows
+ compilation
+
+2015-03-20 08:44 eaxelson
+
+ * back-ends/foma/apply.c, back-ends/foma/constructions.c,
+ back-ends/foma/determinize.c, back-ends/foma/fomalib.h,
+ back-ends/foma/fomalibconf.h, back-ends/foma/lex.cmatrix.c,
+ back-ends/foma/lex.yy.c, back-ends/foma/mem.c,
+ back-ends/foma/minimize.c, back-ends/foma/structures.c,
+ back-ends/openfstwin/src/include/fst/compat.h,
+ back-ends/openfstwin/src/include/fst/properties.h: Added support
+ for compiling openfst and foma back-ends with cl.exe on windows.
+
+2015-03-15 17:45 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll: Include zero in signed and
+ unsigned integers
+
+2015-03-15 16:16 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc:
+ Correction to handling trailing nonmatching material in
+ locate_mode
+
+2015-03-11 14:22 eaxelson
+
+ * libhfst/src/HfstApply.cc, libhfst/src/HfstTransducer.cc: Forgot
+ to commit these files too...
+
+2015-03-11 14:21 eaxelson
+
+ * tools/src/hfst-concatenate.cc, tools/src/hfst-conjunct.cc,
+ tools/src/hfst-disjunct.cc, tools/src/hfst-head.cc,
+ tools/src/hfst-info.cc, tools/src/hfst-minimize.cc,
+ tools/src/hfst-split.cc, tools/src/hfst-subtract.cc,
+ tools/src/hfst-tail.cc: Added xfsm support for some command line
+ tools.
+
+2015-03-11 10:28 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Now HfstTransducer
+ functions throw a FunctionNotImplementedExeption if xfsm
+ implementation is not available.
+
+2015-03-10 15:47 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Added more xfsm
+ implementations.
+
+2015-03-10 14:54 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Added more
+ implementations for XFSM_TYPE for HfstTransducer functions.
+
+2015-03-09 12:41 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Add ignoring
+
+2015-03-09 11:20 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Correct parsing order of
+ PARALLEL_RULES
+
+2015-03-04 15:18 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/ConvertTransducerFormat.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h,
+ tools/src/hfst-compose.cc: Added support for composition for xfsm
+ transducers.
+
+2015-03-03 14:48 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h,
+ tools/src/hfst-minimize.cc: Added a variable to control if (xfsm)
+ transducers that are already minimal are still minimized for
+ profiling purposes.
+
+2015-03-02 15:13 eaxelson
+
+ * libhfst/src/HfstApply.cc,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ tools/src/hfst-fst2txt.cc, tools/src/hfst-minimize.cc,
+ tools/src/hfst-txt2fst.cc: Fixed some bugs in escaping special
+ symbols in prolog format. Added support for xfsm format in some
+ command line tools.
+
+2015-03-02 11:29 eaxelson
+
+ * tools/src/hfst-fst2txt.cc: Fixed an error noticed when
+ -Werror=format-security was enabled during compilation.
+
+2015-02-27 15:00 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h,
+ tools/src/hfst-commandline.cc, tools/src/hfst-fst2fst.cc,
+ tools/src/hfst-fst2txt.cc, tools/src/hfst-txt2fst.cc: Now xfsm
+ conversion works in hfst-fst2fst and between prolog and binary
+ format in hfst-fst2txt and hfst-txt2fst.
+
+2015-02-27 09:55 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h:
+ Added XFSM initializer class.
+
+2015-02-27 09:52 eaxelson
+
+ * libhfst/src/HfstApply.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Added function
+ minimize to xfsm interface
+
+2015-02-26 10:56 eaxelson
+
+ * tools/src/hfst-commandline.cc,
+ tools/src/hfst-compose-intersect.cc, tools/src/hfst-fst2fst.cc:
+ Now hfst-compose-intersect does NOT harmonize transducers by
+ default, as harmonizing produces unexpected results with
+ hyperminimized transducers. Option --harmonize turns on
+ harmonization. This is a partial fix to bug #288.
+
+2015-02-25 11:22 eaxelson
+
+ * configure.ac, libhfst/src/Makefile.am,
+ tools/src/hfst-commandline.cc, tools/src/parsers/XfstCompiler.cc:
+ Added a cross-build Windows patch given in bug report #289.
+
+2015-02-24 21:29 eaxelson
+
+ * libhfst/src/HfstOutputStream.cc: Added a missing #if HAVE_XFSM
+ statement.
+
+2015-02-24 18:43 eaxelson
+
+ * libhfst/src/HfstInputStream.cc, libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc: Fixed some bugs in
+ xfsm stream handling.
+
+2015-02-24 17:15 eaxelson
+
+ * libhfst/src/HfstOutputStream.cc, libhfst/src/HfstOutputStream.h,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Added more
+ functions for writing and reading xfsm transducers.
+
+2015-02-23 14:33 eaxelson
+
+ * libhfst/src/HfstInputStream.h, libhfst/src/HfstOutputStream.cc,
+ libhfst/src/HfstOutputStream.h,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Tentatively added
+ input and output stream functions for xfsm transducer.
+
+2015-02-20 15:12 eaxelson
+
+ * scripts/generate-static-binaries.sh, scripts/hfst-twolc-bin,
+ scripts/package-static-binaries.sh: Modified and added scripts
+ for creating static binaries.
+
+2015-02-18 17:13 mpsilfve
+
+ * tools/src/hfst-compose-intersect.cc: Fast compose intersect
+ available using option -f
+
+2015-02-18 14:52 eaxelson
+
+ * libhfst/src/implementations/ConvertXfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Reorganized xfsm
+ interface.
+
+2015-02-18 14:14 mpsilfve
+
+ * tools/src/hfst-compose-intersect.cc: Hopefully speed up
+ compose_intersect by doing l .o. (l.2 .o. r) instead of l .o. r
+ directly.
+
+2015-02-16 12:47 eaxelson
+
+ * scripts/generate-static-binaries.sh: Updated script.
+
+2015-02-16 12:45 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Disable hfst-xfst's
+ auto-complet with rl_insert instead of rl_abort, which is not
+ found on Mac.
+
+2015-02-16 11:54 eaxelson
+
+ * scripts/generate-static-binaries.sh: Added a script to generate
+ static binaries.
+
+2015-02-16 08:06 eaxelson
+
+ * libhfst/src/implementations/ConvertXfsmTransducer.cc: Now xfsm
+ conversion swaps the state numbers to make a more readable state
+ numbering.
+
+2015-02-13 14:08 eaxelson
+
+ * libhfst/src/implementations/ConvertXfsmTransducer.cc: Now special
+ symbols should also work in xfsm conversion.
+
+2015-02-13 13:01 eaxelson
+
+ * libhfst/src/implementations/ConvertXfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Xfsm conversion
+ functions almost work, escaped symbols still need some attention.
+
+2015-02-12 12:05 eaxelson
+
+ * libhfst/src/implementations/ConvertXfsmTransducer.cc: Conversion
+ functions between xfsm and HfstBasicTransducer formats
+ implemented, alphabets and special symbols still need some work.
+
+2015-02-11 08:36 hardwick
+
+ * tools/src/hfst-optimized-lookup.h: Autoindent entire file after
+ removing tab characters to appease commit script
+
+2015-02-11 08:26 hardwick
+
+ * tools/src/hfst-optimized-lookup.cc,
+ tools/src/hfst-optimized-lookup.h: Handle single-char ascii
+ symbols shadowing multichar ones
+
+2015-02-10 14:14 eaxelson
+
+ * libhfst/src/HfstApply.cc, libhfst/src/HfstTransducer.cc,
+ libhfst/src/HfstTransducer.h, libhfst/src/hfst_apply_schemas.h,
+ libhfst/src/implementations/ConvertXfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Modified hfst to
+ support xfsm format.
+
+2015-02-10 09:15 hardwick
+
+ * tools/src/hfst-proc2.cc: Fix cg quote interpolation
+
+2015-02-10 08:57 hardwick
+
+ * tools/src/hfst-proc2.cc: Clarify help string
+
+2015-02-10 08:55 hardwick
+
+ * tools/src/hfst-proc2.cc: Add segmenting and cg mode; improve
+ handling of special cases and weights
+
+2015-02-09 13:43 eaxelson
+
+ * libhfst/src/HfstDataTypes.cc, libhfst/src/HfstDataTypes.h,
+ libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/ConvertXfsmTransducer.cc,
+ libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/XfsmTransducer.h: Tentatively
+ modified hfst to handle xfsm implementation type.
+
+2015-02-09 13:02 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ tools/src/hfst-pmatch.cc, tools/src/hfst-proc2.cc: Preserve
+ nonmatching sequances in locate_mode and parse the appropriately
+ downstream
+
+2015-02-06 16:51 eaxelson
+
+ * configure.ac, libhfst/src/Makefile.am,
+ libhfst/src/implementations/ConvertTransducerFormat.h,
+ libhfst/src/implementations/ConvertXfsmTransducer.cc,
+ libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/XfsmTransducer.cc,
+ libhfst/src/implementations/XfsmTransducer.h: Tentatively added a
+ skeleton implementation for xfsm library that can be included
+ with configure option --with-xfsm (default is no). No linking to
+ the library is performed, this is just a test version.
+
+2015-02-06 16:34 eaxelson
+
+ * tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc,
+ tools/src/hfst-twolc/src/alphabet_src/Alphabet.h,
+ tools/src/hfst-twolc/src/htwolcpre3.yy,
+ tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc: Added a
+ preprocessing directive that defines hfst-twols's Alphabet as
+ TwolCAlphabet if HAVE_XFSM is on. This will prevent variable
+ collisions.
+
+2015-02-06 13:15 eaxelson
+
+ * tools/src/hfst-txt2fst.cc: Changed hfst-txt2fst's read_prolog
+ into read_prolog_format to avoid conflict with xfsm library
+ function.
+
+2015-02-05 16:07 eaxelson
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/xre_lex.ll, libhfst/src/parsers/xre_parse.yy:
+ Named tokens 'UPPER' and 'LOWER' in pmatch and xre parsers into
+ 'XRE_UPPER', 'XRE_LOWER', 'PMATCH_UPPER' and 'PMATCH_LOWER'
+ because the xfsm library (that might be added to hfst at some
+ point) has an enumerator with the same name in C namespace.
+
+2015-02-05 15:58 eaxelson
+
+ * back-ends/foma/fomalib.h, back-ends/foma/iface.c,
+ back-ends/foma/io.c: Named foma's write_prolog into
+ foma_write_prolog because the xfsm library (that might be added
+ to hfst at some point) has a function with the same name in C
+ namespace.
+
+2015-02-05 15:50 eaxelson
+
+ * tools/src/parsers/xfst-parser.yy: Took away unused token 'LOWER'
+ from xfst parser.
+
+2015-02-03 17:37 eaxelson
+
+ * tools/src/hfst-compose-intersect.cc: Added an option
+ --do-not-harmonize to hfst-compose-intersect.
+
+2015-02-03 16:51 eaxelson
+
+ * tools/src/hfst-compose-intersect.cc: Now hfst-compose-intersect
+ harmonizes the rule transducers with the lexicon.
+
+2015-02-03 15:42 eaxelson
+
+ * tools/src/hfst-fst2strings.cc: Added option --beam to
+ hfst-strings2fst.
+
+2015-02-03 09:39 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.h,
+ tools/src/hfst-pmatch.cc: More accurately name set_locate_mode as
+ set_extract_tags mode
+ (locate vs. match is determined by function call)
+
+2015-02-02 14:29 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Fix bug in compiling
+ UNKNOWNs on the left side of pair separators
+
+2015-02-02 13:58 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Fix bug
+ affecting locate-mode where ?-matches were being seen as
+ IDENTITIES in input.
+
+2015-02-02 10:34 eaxelson
+
+ * tools/src/hfst-lookup.cc: Added option --beam also to
+ hfst-lookup. It is mostly untested for this tool.
+
+2015-01-30 13:35 eaxelson
+
+ * tools/src/hfst-optimized-lookup.cc: Tentatively added option
+ --beam to hfst-optimized-lookup.
+
+2015-01-30 11:30 eaxelson
+
+ * tools/src/hfst-compose-intersect.cc: Fixed a typo in warning
+ message about missing symbols in input tapes of rule transducers
+ ('output' changed to 'input').
+
+2015-01-28 10:19 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: In locate
+ mode, zero-length hits could cause infinite loops. Commit fixes
+ it.
+
+2015-01-23 15:31 eaxelson
+
+ * tools/src/hfst-lookup.cc: Now hfst-lookup checks it it is
+ possible for an input to go through a transducer before calling
+ is_infinitely_ambiguous. Should fix bug #278.
+
+2015-01-22 11:40 eaxelson
+
+ * libhfst/src/HfstTransducer.cc: Now reserved symbols are detected
+ in composition if xerox-composition is ON. This will prevent
+ symbol collisions by throwing an error message, giving at least a
+ temporary solution to issues with reserved symbols.
+
+2015-01-22 09:52 eaxelson
+
+ * libhfst/src/parsers/xre_parse.yy: Fixed substitution operator in
+ regexp, now flag diacritics are allowed both as substituting and
+ subtituted symbols. Should fix bug #284.
+
+2015-01-21 21:31 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/xfst-parser.yy: Added better support for
+ special symbols in hfst-xfst's substitute command.
+
+2015-01-21 15:25 eaxelson
+
+ * tools/src/parsers/test/Makefile.am,
+ tools/src/parsers/test/substitute_symbol_7.att,
+ tools/src/parsers/test/substitute_symbol_7.xfst,
+ tools/src/parsers/test/substitute_symbol_8.att,
+ tools/src/parsers/test/substitute_symbol_8.xfst,
+ tools/src/parsers/test/test.sh: Added two tests for hfst-xfst's
+ substitute command. They are skipped until substitute can also
+ handle them.
+
+2015-01-21 12:48 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/parsers/LexcCompiler.cc: Fixed a bug in composition
+ when xerox-composition is ON. Flag diacritics @SOMEFLAG@ were
+ earlier transformed into $ and back into @SOMEFLAG@, but symbols
+ of form $...$ are already reserved for lexc. Now diacritics are
+ escaped as %SOMEFLAG% during composition. This creates a new set
+ of reserved symbols which maybe needs more consideration
+ though... Also added debug prints for LexcCompiler.
+
+2015-01-16 14:13 eaxelson
+
+ * configure.ac: Removed xml checks from configure, since they cause
+ unlinked xml references in hfst-edit-metadata on some platforms.
+
+2015-01-16 13:13 eaxelson
+
+ * libhfst/src/HfstFlagDiacritics.cc,
+ tools/src/parsers/XfstCompiler.cc: Fixed flag diacritic
+ recognition, now also flags of form e.g. '@D.FOO@' are accepted
+ and an empty string is returned as their value. Also added full
+ support for flag handling in hfst-xfst.
+
+2015-01-15 15:55 eaxelson
+
+ * libhfst/src/HfstSymbolDefs.cc, libhfst/src/HfstSymbolDefs.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/XfstCompiler.h: Added some conversion functions
+ between StringPairVectors and StringVectors. Added checks for
+ flag diacritics in hfst-xfst non-optimized lookup.
+
+2015-01-08 14:33 eaxelson
+
+ * tools/src/hfst-summarize.cc: A small fix to hfst-summarize option
+ handling.
+
+2014-12-19 15:34 eaxelson
+
+ * ChangeLog, ChangeLog.old, NEWS, configure.ac,
+ libhfst/src/Makefile.am: Ready for release 3.8.2.
+
+2014-12-19 14:41 eaxelson
+
+ * tools/src/hfst-summarize.cc: Added option
+ --print-symbol-pair-statistics(=N) to hfst-summarize.
+
2014-12-17 13:28 eaxelson
* configure.ac, libhfst/src/HfstTransducer.cc,
diff --git a/ChangeLog.old b/ChangeLog.old
index 56e3859..50487c4 100644
--- a/ChangeLog.old
+++ b/ChangeLog.old
@@ -1,3 +1,427 @@
+2014-12-17 13:28 eaxelson
+
+ * configure.ac, libhfst/src/HfstTransducer.cc,
+ libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/xre_lex.ll,
+ tools/src/parsers/XfstCompiler.cc: Small fixes to
+ compile-replace. Also restarting the char counter every time a
+ new xre parsing is started. Removed the xml2 dependecy in
+ configure.
+
+2014-12-12 10:49 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Omit
+ unnecessary step in stringification
+
+2014-12-12 10:22 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Another
+ slight speed improvement
+
+2014-12-12 09:56 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Some
+ double free insurance related to previous commit
+
+2014-12-12 09:53 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Some more
+ pmatch runtime speedup (around 5-10% in most cases)
+
+2014-12-11 21:54 hardwick
+
+ * libhfst/src/implementations/ConvertOlTransducer.cc: Remove
+ commented-out lines (same ones I was intending in the previous
+ commit)
+
+2014-12-11 21:08 hardwick
+
+ * libhfst/src/implementations/ConvertOlTransducer.cc: Remove
+ extraenous index table fitting test
+
+2014-12-11 20:33 hardwick
+
+ * libhfst/src/implementations/ConvertOlTransducer.cc,
+ libhfst/src/implementations/optimized-lookup/convert.h: Further
+ improvements to conversion to optimized lookup format
+
+ This speeds up conversion a bit more, and also improves packing
+ in one case by around 25% (implying that this commit *does*
+ change the
+ output of the conversion, but in a functionally equivalent way).
+
+ The space gain was mainly accomplished by changing what happens
+ when we fail
+ to find a suitable index table location several times a row
+ starting in the
+ same place. We used to jump up to the previous successful
+ location and its
+ indices, now we keep trying to fill in the gaps between those
+ first.
+
+ This commit also removes the state-relabeling facility, because
+ it didn't seem
+ to be doing anything (perhaps something in HfstBasicTransducer
+ has changed;
+ it seems state numbers can no longer be non-contiguous). So we
+ now assume
+ states are contiguous and iterated in order. If this assumption
+ is broken,
+ this needs to be changes. But as I said earlier, looks like
+ relabeling wasn't
+ doing anything now anyway.
+
+2014-12-11 13:25 hardwick
+
+ * libhfst/src/implementations/ConvertOlTransducer.cc,
+ libhfst/src/implementations/optimized-lookup/convert.cc,
+ libhfst/src/implementations/optimized-lookup/convert.h: Large
+ speedup in conversion to optimized-lookup format
+
+ In some cases this speeds up conversion by up to 80%. This
+ involved
+ a) making the code a bit more convoluted by using multiple data
+ structures
+ where there used to be just one
+ b) omitting some space-oriented optimizations that appear to be
+ usually
+ very minor and occasionally even harmful
+
+ While I've tested this with the major transducers I've found,
+ it's not
+ unthinkable that refactoring such hairy code as this has
+ introduced bugs.
+ No new functionality is introduced so it's ok to roll this back
+ if it causes
+ problems.
+
+2014-12-05 16:03 eaxelson
+
+ * tools/src/parsers/test/Makefile.am,
+ tools/src/parsers/test/compile_replace_1.output,
+ tools/src/parsers/test/compile_replace_1.xfst,
+ tools/src/parsers/test/compile_replace_2.output,
+ tools/src/parsers/test/compile_replace_2.xfst,
+ tools/src/parsers/test/compile_replace_3.output,
+ tools/src/parsers/test/compile_replace_3.xfst,
+ tools/src/parsers/test/test.sh: Added test cases for
+ compile-replace.
+
+2014-12-05 14:49 eaxelson
+
+ * tools/src/parsers/test/Makefile.am,
+ tools/src/parsers/test/merge.att,
+ tools/src/parsers/test/merge.xfst,
+ tools/src/parsers/test/merge_weighted.att,
+ tools/src/parsers/test/merge_weighted.xfst,
+ tools/src/parsers/test/test.sh: Added test cases for merge
+ operation.
+
+2014-12-05 12:22 eaxelson
+
+ * tools/src/hfst-fst2strings.cc: Fixed an error in hfst-fst2strings
+ --print-separator where two consecutive lines of -- were printed
+ between non-empty transducers in some cases.
+
+2014-12-04 13:04 eaxelson
+
+ * libhfst/src/parsers/xre_utils.cc,
+ tools/src/parsers/XfstCompiler.cc: Now minimizing the merging
+ automaton before merge operation so that epsilons do not cut a
+ succesfull merge path. Also allowing epsilon-to-regexp-marker
+ transitions in the merge filter.
+
+2014-12-03 15:35 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ tools/src/parsers/XfstCompiler.cc: Improvements to
+ compile-replace function, now it should work for input and output
+ sides of a transducer.
+
+2014-12-03 09:36 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Revert bungled change to
+ precedence order
+
+2014-12-02 20:34 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/xre_lex.ll, tools/src/parsers/hfst-xfst.cc:
+ Now xre compiler of function merge does not increment the char
+ counter, making it possible to have many merge operators inside
+ one regex.
+
+2014-12-02 16:07 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/xre_utils.cc: Added a constructor
+ XreCompiler(XreConstructorArguments & args) to facilitate passing
+ xre variables to merge function which needs them in its internal
+ xre compiler.
+
+2014-12-02 14:06 eaxelson
+
+ * libhfst/src/parsers/xre_utils.cc: Now using internal starptr
+ variables in functions hfst::xre::compile and
+ hfst::xre::compile_first instead of global hfst::xre::startptr.
+ This should fix the strange memory errors which occurred when
+ calling merge operation inside a regular expression.
+
+2014-12-02 12:33 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/HfstTransitionGraph.h: Now merge
+ operation filters out non-optimal paths.
+
+2014-11-27 14:49 eaxelson
+
+ * libhfst/src/HfstTokenizer.cc, libhfst/src/HfstTokenizer.h:
+ Tentatively added a function 'tokenize_and_align_flag_diacritics'
+ to HfstTokenizer.
+
+2014-11-26 16:50 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Runtime
+ speed improvements
+ Prereserve table vectors, eliminate special_symbols map
+
+2014-11-26 13:00 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ libhfst/src/implementations/optimized-lookup/transducer.h,
+ libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h, tools/src/hfst-pmatch.cc,
+ tools/src/hfst-proc2.cc: Profiling support with Counter() and
+ --profile & a bunch of smaller changes
+
+ Counter(arg) in a ruleset inserts a profiling point, hfst-pmatch
+ --profile
+ prints profiling info.
+
+ When --verbose, warn about undefined symbols being interpreted as
+ labels.
+ When --flatten, flatten Lst() definitions too.
+
+2014-11-25 14:14 eaxelson
+
+ * tools/src/hfst-compose.cc: Allowing 1-to-n composition of
+ automata in archives. Fixes bug (or feature request) #277.
+
+2014-11-25 10:54 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/LexcCompiler.h,
+ libhfst/src/parsers/lexc-lexer.ll,
+ libhfst/src/parsers/lexc-parser.yy,
+ tools/src/hfst-lexc-compiler.cc: Now passing verbosity to
+ LexcCompiler as an unsigned integer via setVerbosity(uint). Also
+ made small fixes to warning prints in lexc compiler.
+
+2014-11-24 13:51 eaxelson
+
+ * test/tools/Makefile.am,
+ test/tools/lexc-compiler-functionality.sh,
+ test/tools/warn.one-sided-flags.lexc,
+ test/tools/warn.one-sided-flags.lexc.flag.result,
+ test/tools/warn.one-sided-flags.lexc.result: Added tests for
+ one-sided flag diacritics for hfst-lexc.
+
+2014-11-24 13:50 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/lexc-parser.yy: Now lexc compiler warns about
+ one-sided flag diacritics in verbose mode.
+
+2014-11-24 13:49 eaxelson
+
+ * libhfst/src/HfstTokenizer.cc, libhfst/src/HfstTokenizer.h: Added
+ a tokenizing function that warns about symbol pairs, if needed.
+
+2014-11-24 11:07 eaxelson
+
+ * test/tools/Makefile.am,
+ test/tools/lexc-compiler-functionality.sh,
+ test/tools/xfail.sublexicon-defined-more-than-once.lexc: Added
+ test case for previous lexc commit (sublexicon defined more than
+ once treated as an error).
+
+2014-11-24 10:33 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/LexcCompiler.h,
+ libhfst/src/parsers/lexc-parser.yy: Now multiple definitions of
+ the same lexicon in lexc are treated as an error unless
+ LexcCompiler::setAllowMultipleLexiconDefinitions(true) is called
+ first.
+
+2014-11-24 09:04 eaxelson
+
+ * libhfst/src/parsers/lexc-lexer.ll,
+ libhfst/src/parsers/lexc-parser.yy: Now lexc parser updates the
+ error status hlexcnerrs when hlexcerror is called. In case of
+ warnings, the error status is nor updated.
+
+2014-11-20 12:15 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ libhfst/src/implementations/optimized-lookup/transducer.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Speed
+ up list arc processing by replacing some maps and sets with
+ vectors
+
+2014-11-19 09:17 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/xre_parse.yy,
+ libhfst/src/parsers/xre_utils.cc,
+ libhfst/src/parsers/xre_utils.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/XfstCompiler.h: Changed list to set in merge
+ operation. Removed commented code.
+
+2014-11-18 17:00 eaxelson
+
+ * tools/src/hfst-lexc-compiler.cc: Added option --encode-weights to
+ hfst-lexc.
+
+2014-11-17 13:55 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Reinstate undefined symbols
+ as valid tokens as per documentation
+
+2014-11-17 13:53 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc: Add defined lists
+
+2014-11-17 13:14 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Add Sigma()
+
+2014-11-17 11:48 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Added Lst() and support for
+ list arcs in runtime
+
+2014-11-17 11:21 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Sync precedence rules
+
+2014-11-17 11:17 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Switch order of precedence
+ of concatenation and other binary operations
+
+2014-11-17 07:47 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Added Lit()
+
+2014-11-13 07:21 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/HfstTransitionGraph.h: Modified the
+ function merge, it now takes as an argument a map of list
+ symbols.
+
+2014-11-13 07:16 eaxelson
+
+ * libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/xre_parse.yy,
+ libhfst/src/parsers/xre_utils.cc,
+ libhfst/src/parsers/xre_utils.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/test/Makefile.am,
+ tools/src/parsers/test/one_transition_regex.att,
+ tools/src/parsers/test/one_transition_regex.xfst,
+ tools/src/parsers/test/test.sh: Fixed a bug in xre parser, now
+ definitions and unknowns can be used together in expressions such
+ as 'regex [def:?] ;'.
+
+2014-11-11 14:06 eaxelson
+
+ * libhfst/src/HfstTokenizer.cc,
+ tools/src/HfstStrings2FstTokenizer.cc: Now the epsilon symbol is
+ not added as a multichar symbol to hfst-lookup tokenizer if it is
+ the empty string. Should fix bug #275.
+
+2014-11-10 21:26 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_utils.cc: Allow \U00NNNNNN syntax for
+ code points in utf-8 but above U+FFFF
+
+2014-11-10 20:55 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_utils.cc: Allow \UNNNN as well as
+ \uNNNN
+
+2014-11-10 20:41 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_utils.cc: Allow \uNNNN in range
+ notation and make some fixes to utf-8 handling
+
+2014-11-10 18:58 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Added utf-8 character range
+ expressions
+
+2014-11-10 14:57 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/parsers/xre_lex.ll, libhfst/src/parsers/xre_parse.yy:
+ An untested implementation of the merge operation added to
+ hfst-xfst.
+
+2014-11-10 13:23 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Syntax-level completion of
+ functions (arg placement still not completely free)
+
+2014-11-04 16:35 eaxelson
+
+ * check_installation/copy-tool-tests.sh: Fixed a small bug in hfst
+ tool tester.
+
+2014-11-04 10:23 eaxelson
+
+ * ChangeLog, ChangeLog.old, NEWS, configure.ac,
+ libhfst/src/Makefile.am, swig/setup.py: Ready for release 3.8.1.
+
2014-10-31 16:44 eaxelson
* tools/src/hfst-guess.cc: Fixed std::cout into &std::cout in
diff --git a/NEWS b/NEWS
index e937203..2f6d1ed 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,15 @@
This file contains all noteworthy changes in HFST development between releases.
For full listing of changes see ChangeLog.
+Noteworthy changes in 3.8.3
+---------------------------
+
+* Bugfixes and improvements to many tools, including hfst-xfst, hfst-proc(2),
+ hfst-lexc, hfst-pmatch(2fst), hfst-reweight, hfst-pair-test, hfst-compose-intersect
+
+* Better support for command line tools on windows
+
+
Noteworthy changes in 3.8.2
---------------------------
@@ -591,4 +600,3 @@ Noteworthy changes in release 2.1
* hfst-twolc and hfst-compose-intersect understand # like xerox originals
-
diff --git a/back-ends/foma/Makefile.am b/back-ends/foma/Makefile.am
index f545c68..90a9d4d 100644
--- a/back-ends/foma/Makefile.am
+++ b/back-ends/foma/Makefile.am
@@ -5,7 +5,7 @@ AUTOMAKE_OPTIONS=std-options
AM_CPPFLAGS= -Wno-deprecated -std=c99 -D_XOPEN_SOURCE=500
if WANT_MINGW
-AM_CPPFLAGS += -DWINDOWS -D__NO_MINGW_LFS
+AM_CPPFLAGS += -D__NO_MINGW_LFS
endif
noinst_LTLIBRARIES = libfoma.la
diff --git a/back-ends/foma/apply.c b/back-ends/foma/apply.c
index b2358dd..91f9c21 100644
--- a/back-ends/foma/apply.c
+++ b/back-ends/foma/apply.c
@@ -393,10 +393,9 @@ void apply_clear_index(struct apply_handle *h) {
}
void apply_index(struct apply_handle *h, int inout, int densitycutoff, int mem_limit, int flags_only) {
- struct fsm_state *fsm;
+ struct fsm_state *fsm = h->gstates; // must be defined in the beginning for cl.exe compiler
unsigned int cnt = 0;
int i, j, maxtrans, numtrans, laststate, sym, stateno;
- fsm = h->gstates;
struct apply_state_index **indexptr, *iptr, *tempiptr;
diff --git a/back-ends/foma/constructions.c b/back-ends/foma/constructions.c
index e6564c9..51b49bb 100644
--- a/back-ends/foma/constructions.c
+++ b/back-ends/foma/constructions.c
@@ -45,7 +45,7 @@ int sort_cmp(const void *a, const void *b) {
return (((const struct fsm_state *)a)->state_no - ((const struct fsm_state *)b)->state_no);
}
-inline int add_fsm_arc(struct fsm_state *fsm, int offset, int state_no, int in, int out, int target, int final_state, int start_state);
+INLINE int add_fsm_arc(struct fsm_state *fsm, int offset, int state_no, int in, int out, int target, int final_state, int start_state);
struct fsm *fsm_kleene_closure(struct fsm *net, int optionality);
diff --git a/back-ends/foma/determinize.c b/back-ends/foma/determinize.c
index d48bf3b..85eb9fd 100644
--- a/back-ends/foma/determinize.c
+++ b/back-ends/foma/determinize.c
@@ -79,8 +79,8 @@ static struct nhash_list *table;
extern int add_fsm_arc(struct fsm_state *fsm, int offset, int state_no, int in, int out, int target, int final_state, int start_state);
static void init(struct fsm *net);
-inline static int e_closure(int states);
-inline static int set_lookup(int *lookup_table, int size);
+INLINE static int e_closure(int states);
+INLINE static int set_lookup(int *lookup_table, int size);
static int initial_e_closure(struct fsm *network);
static void memoize_e_closure(struct fsm_state *fsm);
static int next_unmarked(void);
@@ -88,7 +88,7 @@ static void single_symbol_to_symbol_pair(int symbol, int *symbol_in, int *symbol
static int symbol_pair_to_single_symbol(int in, int out);
static void sigma_to_pairs(struct fsm *net);
static int nhash_find_insert(int *set, int setsize);
-inline static int hashf(int *set, int setsize);
+INLINE static int hashf(int *set, int setsize);
static int nhash_insert(int hashval, int *set, int setsize);
static void nhash_rebuild_table ();
static void nhash_init (int initial_size);
@@ -434,7 +434,7 @@ static void init_trans_array(struct fsm *net) {
}
}
-inline static int e_closure(int states) {
+INLINE static int e_closure(int states) {
int i, set_size;
struct e_closure_memo *ptr;
@@ -493,7 +493,7 @@ inline static int e_closure(int states) {
return(set_lookup(temp_move, set_size));
}
-inline static int set_lookup (int *lookup_table, int size) {
+INLINE static int set_lookup (int *lookup_table, int size) {
/* Look up a set and its corresponding state number */
/* if it doesn't exist from before, assign a state number */
@@ -742,7 +742,7 @@ static int nhash_find_insert(int *set, int setsize) {
}
}
-inline static int hashf(int *set, int setsize) {
+INLINE static int hashf(int *set, int setsize) {
int i;
unsigned int hashval, sum = 0;
hashval = 6703271;
diff --git a/back-ends/foma/fomalib.h b/back-ends/foma/fomalib.h
index 6a1ecd4..c4871c3 100644
--- a/back-ends/foma/fomalib.h
+++ b/back-ends/foma/fomalib.h
@@ -26,7 +26,13 @@ extern "C" {
#include <zlib.h>
#endif
-#define FEXPORT __attribute__((visibility("default")))
+#ifdef _MSC_VER
+ #define FEXPORT __declspec(dllexport)
+ #define INLINE
+#else
+ #define FEXPORT __attribute__((visibility("default")))
+ #define INLINE inline
+#endif
/* Library version */
#define MAJOR_VERSION 0
@@ -302,7 +308,8 @@ FEXPORT int fsm_write_binary_file(struct fsm *net, char *filename);
FEXPORT int load_defined(char *filename);
FEXPORT int save_defined();
FEXPORT int save_stack_att();
-FEXPORT int write_prolog(struct fsm *net, char *filename);
+ // write_prolog changed to foma_write_prolog because it conflicts with an xfsm function
+FEXPORT int foma_write_prolog(struct fsm *net, char *filename);
#ifdef ZLIB // HFST addition
FEXPORT int foma_net_print(struct fsm *net, gzFile *outfile);
#endif
@@ -369,6 +376,13 @@ FEXPORT int fsm_construct_check_symbol(struct fsm_construct_handle *handle, char
FEXPORT void fsm_construct_copy_sigma(struct fsm_construct_handle *handle, struct sigma *sigma);
FEXPORT struct fsm *fsm_construct_done(struct fsm_construct_handle *handle);
+#ifndef __cplusplus
+#ifndef bool
+ #define bool int
+ #define false((bool)0)
+ #define true((bool)1)
+#endif
+#endif
/******************/
/* String hashing */
diff --git a/back-ends/foma/fomalibconf.h b/back-ends/foma/fomalibconf.h
index cf3b97c..e6d1d39 100644
--- a/back-ends/foma/fomalibconf.h
+++ b/back-ends/foma/fomalibconf.h
@@ -15,6 +15,16 @@
/* You should have received a copy of the GNU General Public License */
/* along with foma. If not, see <http://www.gnu.org/licenses/>. */
+#ifndef __cplusplus
+#ifndef bool
+ #define bool int
+ #define false ((bool)0)
+ #define true ((bool)1)
+#endif
+#endif
+
+#define _Bool bool
+
struct state_array {
struct fsm_state *transitions;
};
diff --git a/back-ends/foma/iface.c b/back-ends/foma/iface.c
index 67e367f..904c7a0 100644
--- a/back-ends/foma/iface.c
+++ b/back-ends/foma/iface.c
@@ -24,7 +24,9 @@
#include <ctype.h>
#include "foma.h"
+#ifdef ZLIB
#include "zlib.h"
+#endif
extern int g_show_flags;
extern int g_obey_flags;
@@ -1012,7 +1014,7 @@ void iface_save_stack(char *filename) {
printf("Writing to file %s.\n", filename);
for (stack_ptr = stack_find_bottom(); stack_ptr->next != NULL; stack_ptr = stack_ptr->next) {
#ifdef ZLIB
- foma_net_print(stack_ptr->fsm, outfile);
+ foma_net_print(stack_ptr->fsm, outfile);
#endif
}
gzclose(outfile);
@@ -1271,7 +1273,7 @@ int iface_write_att(char *filename) {
void iface_write_prolog(char *filename) {
if (iface_stack_check(1))
- write_prolog(stack_find_top()->fsm, filename);
+ foma_write_prolog(stack_find_top()->fsm, filename);
}
void iface_zero_plus() {
diff --git a/back-ends/foma/io.c b/back-ends/foma/io.c
index b4664e8..e5d5d59 100644
--- a/back-ends/foma/io.c
+++ b/back-ends/foma/io.c
@@ -98,12 +98,13 @@ int load_defined(char *filename) {
io_error(); return 0; }
int save_defined(char *filename) {
io_error(); return 0; }
-int write_prolog (struct fsm *net, char *filename) {
+// write_prolog changed to foma_write_prolog because it conflicts with an xfsm function
+int foma_write_prolog(struct fsm *net, char *filename) {
io_error(); return 0; }
#else
// HFST addition
-#ifdef WINDOWS
+#if defined (_MSC_VER) || (__MINGW32__)
#define LONG_LONG_SPECIFIER "%I64d"
#else
#define LONG_LONG_SPECIFIER "%lld"
@@ -124,7 +125,8 @@ void escape_print(FILE *stream, char* string) {
}
}
-int write_prolog (struct fsm *net, char *filename) {
+// write_prolog changed to foma_write_prolog because it conflicts with an xfsm function
+int foma_write_prolog(struct fsm *net, char *filename) {
struct fsm_state *stateptr;
int i, *finals, *used_symbols, maxsigma;
FILE *out;
diff --git a/back-ends/foma/lex.cmatrix.c b/back-ends/foma/lex.cmatrix.c
index d5add37..750d0bf 100644
--- a/back-ends/foma/lex.cmatrix.c
+++ b/back-ends/foma/lex.cmatrix.c
@@ -76,6 +76,7 @@ typedef unsigned int flex_uint32_t;
#endif /* ! C99 */
/* Limits of integral types. */
+#ifndef _MSC_VER
#ifndef INT8_MIN
#define INT8_MIN (-128)
#endif
@@ -103,6 +104,7 @@ typedef unsigned int flex_uint32_t;
#ifndef UINT32_MAX
#define UINT32_MAX (4294967295U)
#endif
+#endif // _MSC_VER
#endif /* ! FLEXINT_H */
@@ -739,7 +741,9 @@ void my_cmatrixparse(struct fsm *net, char *my_string) {
* down here because we want the user's section 1 to have been scanned first.
* The user has a chance to override it with an option.
*/
+#ifndef _MSC_VER
#include <unistd.h>
+#endif // _MSC_VER
#endif
#ifndef YY_EXTRA_TYPE
@@ -1217,7 +1221,7 @@ ECHO;
{
(yy_did_buffer_switch_on_eof) = 0;
- if ( cmatrixwrap( ) )
+ if ( cmatrixwrap( void ) )
{
/* Note: because we've taken care in
* yy_get_next_buffer() to have set up
@@ -1482,7 +1486,7 @@ static int yy_get_next_buffer (void)
case EOB_ACT_END_OF_FILE:
{
- if ( cmatrixwrap( ) )
+ if ( cmatrixwrap( void ) )
return 0;
if ( ! (yy_did_buffer_switch_on_eof) )
diff --git a/back-ends/foma/lex.lexc.c b/back-ends/foma/lex.lexc.c
index 48fe99a..36babcb 100644
--- a/back-ends/foma/lex.lexc.c
+++ b/back-ends/foma/lex.lexc.c
@@ -1,7 +1,7 @@
#line 22 "lexc.l"
#define YY_BUF_SIZE 16777216
-#ifdef WINDOWS
+#if defined (_MSC_VER) || (__MINGW32__)
typedef long off_t;
#endif
diff --git a/back-ends/foma/lex.yy.c b/back-ends/foma/lex.yy.c
index 572dc99..c91e54c 100644
--- a/back-ends/foma/lex.yy.c
+++ b/back-ends/foma/lex.yy.c
@@ -1,4 +1,4 @@
-#ifdef WINDOWS
+#if defined (_MSC_VER) || (__MINGW32__)
typedef long off_t;
#endif
@@ -60,6 +60,7 @@ typedef unsigned int flex_uint32_t;
#endif /* ! C99 */
/* Limits of integral types. */
+#ifndef _MSC_VER
#ifndef INT8_MIN
#define INT8_MIN (-128)
#endif
@@ -87,6 +88,7 @@ typedef unsigned int flex_uint32_t;
#ifndef UINT32_MAX
#define UINT32_MAX (4294967295U)
#endif
+#endif // _MSC_VER
#endif /* ! FLEXINT_H */
@@ -1760,9 +1762,14 @@ void yyset_lineno (int line_number, yyscan_t yyscanner);
int my_yyparse(char *my_string, int lineno) {
int yyp;
yyscan_t scanner;
-
+#ifdef _MSC_VER
+ YY_BUFFER_STATE my_string_buffer = NULL; // initialization must be done here for cl.exe
+#endif
+
yylex_init(&scanner);
- YY_BUFFER_STATE my_string_buffer;
+#ifndef _MSC_VER
+ YY_BUFFER_STATE my_string_buffer; // see above ifdef
+#endif
my_string_buffer = yy_scan_string(my_string,scanner);
yyset_lineno(lineno,scanner);
if (g_parse_depth > 0) {
@@ -1813,7 +1820,9 @@ int my_yyparse(char *my_string, int lineno) {
* down here because we want the user's section 1 to have been scanned first.
* The user has a chance to override it with an option.
*/
+#ifndef _MSC_VER
#include <unistd.h>
+#endif // _MSC_VER
#endif
#ifndef YY_EXTRA_TYPE
diff --git a/back-ends/foma/mem.c b/back-ends/foma/mem.c
index 4d0fd76..4087ead 100644
--- a/back-ends/foma/mem.c
+++ b/back-ends/foma/mem.c
@@ -72,11 +72,11 @@ unsigned int round_up_to_power_of_two(unsigned int v) {
return(v);
}
-inline void *xxmalloc(size_t size) {
+INLINE void *xxmalloc(size_t size) {
return(malloc(size));
}
-inline void xxfree(void *ptr) {
+INLINE void xxfree(void *ptr) {
free(ptr);
}
@@ -84,11 +84,11 @@ void *xxrealloc(void *ptr, size_t size) {
return(realloc(ptr, size));
}
-inline void *xxcalloc(size_t nmemb, size_t size) {
+INLINE void *xxcalloc(size_t nmemb, size_t size) {
return(calloc(nmemb,size));
}
-inline char *xxstrdup(const char *s) {
+INLINE char *xxstrdup(const char *s) {
return(strdup(s));
}
diff --git a/back-ends/foma/minimize.c b/back-ends/foma/minimize.c
index b07fc83..6032337 100644
--- a/back-ends/foma/minimize.c
+++ b/back-ends/foma/minimize.c
@@ -83,12 +83,12 @@ static struct p *P, *Phead, *Pnext, *current_w;
static struct e *E;
static struct agenda *Agenda_head, *Agenda_top, *Agenda_next, *Agenda;
-static inline int refine_states(int sym);
+static INLINE int refine_states(int sym);
static void init_PE();
static void agenda_add(struct p *pptr, int start);
static void sigma_to_pairs(struct fsm *net);
/* static void single_symbol_to_symbol_pair(int symbol, int *symbol_in, int *symbol_out); */
-static inline int symbol_pair_to_single_symbol(int in, int out);
+static INLINE int symbol_pair_to_single_symbol(int in, int out);
static void generate_inverse(struct fsm *net);
struct fsm *fsm_minimize(struct fsm *net) {
@@ -305,7 +305,7 @@ static struct fsm *rebuild_machine(struct fsm *net) {
return(net);
}
-static inline int refine_states(int invstates) {
+static INLINE int refine_states(int invstates) {
int i, selfsplit;
struct e *thise;
struct p *tP, *newP = NULL;
@@ -668,6 +668,6 @@ static void sigma_to_pairs(struct fsm *net) {
num_symbols = x;
}
-static inline int symbol_pair_to_single_symbol(int in, int out) {
+static INLINE int symbol_pair_to_single_symbol(int in, int out) {
return(*(double_sigma_array+maxsigma*in+out));
}
diff --git a/back-ends/foma/sigma.c b/back-ends/foma/sigma.c
index 86d7705..2ac8a81 100644
--- a/back-ends/foma/sigma.c
+++ b/back-ends/foma/sigma.c
@@ -336,9 +336,16 @@ struct ssort {
int number;
};
-int ssortcmp(struct ssort *a, struct ssort *b) {
- return(strcmp(a->symbol, b->symbol));
-}
+
+#ifdef _MSC_VER
+ int ssortcmp(const void *a, const void *b) {
+ return(strcmp(((struct ssort*)a)->symbol, ((struct ssort*)b)->symbol));
+ }
+#else
+ int ssortcmp(struct ssort *a, struct ssort *b) {
+ return(strcmp(a->symbol, b->symbol));
+ }
+#endif // _MSC_VER
struct sigma *sigma_copy(struct sigma *sigma) {
int f = 0;
@@ -367,7 +374,11 @@ struct sigma *sigma_copy(struct sigma *sigma) {
/* and sorts the sigma based on the symbol string contents */
int sigma_sort(struct fsm *net) {
+#ifdef _MSC_VER
+ int(*comp)(const void*, const void*) = ssortcmp;
+#else
int(*comp)() = ssortcmp;
+#endif // _MSC_VER
int size, i, max, *replacearray;
struct ssort *ssort;
struct sigma *sigma;
diff --git a/back-ends/foma/spelling.c b/back-ends/foma/spelling.c
index b01ed71..f5c6227 100644
--- a/back-ends/foma/spelling.c
+++ b/back-ends/foma/spelling.c
@@ -34,7 +34,7 @@
#define BITCLEAR(a,b) ((a)[BITSLOT(b)] &= ~BITMASK(b))
#define BITTEST(a,b) ((a)[BITSLOT(b)] & BITMASK(b))
#define BITNSLOTS(nb) ((nb + CHAR_BIT - 1) / CHAR_BIT)
-#define min(X, Y) ((X) < (Y) ? (X) : (Y))
+#define min_(X, Y) ((X) < (Y) ? (X) : (Y)) // HFST modification: msvc compiler complains about macro redefinition if 'min' is used
static int calculate_h(struct apply_med_handle *medh, int *intword, int currpos, int state);
static struct astarnode *node_delete_min();
@@ -618,7 +618,7 @@ void fsm_create_letter_lookup(struct apply_med_handle *medh, struct fsm *net) {
letterbits_union(v, vp, medh->letterbits,medh->bytes_per_letter_array); /* add v' target bits to v */
letterbits_add(v, curr_ptr->in, medh->letterbits,medh->bytes_per_letter_array); /* add current arc label to v */
- (sccinfo+v)->lowlink = min((sccinfo+v)->lowlink,(sccinfo+vp)->lowlink);
+ (sccinfo+v)->lowlink = min_((sccinfo+v)->lowlink,(sccinfo+vp)->lowlink);
if ((curr_ptr+1)->state_no != curr_ptr->state_no) {
goto l4;
@@ -652,7 +652,7 @@ void fsm_create_letter_lookup(struct apply_med_handle *medh, struct fsm *net) {
/* if v' visited */
/* T: v.lowlink = min(v.lowlink, v'.lowlink), union v.list with e, move to next edge in v, goto loop */
} else if ((sccinfo+vp)->on_t_stack) {
- (sccinfo+v)->lowlink = min((sccinfo+v)->lowlink,(sccinfo+vp)->lowlink);
+ (sccinfo+v)->lowlink = min_((sccinfo+v)->lowlink,(sccinfo+vp)->lowlink);
}
/* If node is visited, copy its bits */
letterbits_union(v,vp,medh->letterbits,medh->bytes_per_letter_array);
diff --git a/back-ends/foma/structures.c b/back-ends/foma/structures.c
index e378ee7..d6e78f8 100644
--- a/back-ends/foma/structures.c
+++ b/back-ends/foma/structures.c
@@ -18,7 +18,9 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
-#include <sys/time.h>
+#ifndef _MSC_VER
+ #include <sys/time.h>
+#endif
#include "foma.h"
static struct defined_quantifiers *quantifiers;
@@ -29,20 +31,36 @@ char *fsm_get_library_version_string() {
return(s);
}
+
+#ifdef _MSC_VER
+int linesortcompin(const void *a, const void *b) {
+ return (((struct fsm_state*)a)->in - ((struct fsm_state*)b)->in);
+}
+int linesortcompout(const void *a, const void *b) {
+ return (((struct fsm_state*)a)->out - ((struct fsm_state*)b)->out);
+}
+#else
int linesortcompin(struct fsm_state *a, struct fsm_state *b) {
- return (a->in - b->in);
+ return (a->in - b->in);
}
-
int linesortcompout(struct fsm_state *a, struct fsm_state *b) {
- return (a->out - b->out);
+ return (a->out - b->out);
}
+#endif // _MSC_VER
void fsm_sort_arcs(struct fsm *net, int direction) {
/* direction 1 = in, direction = 2, out */
struct fsm_state *fsm;
int i, lasthead, numlines;
+
+#ifdef _MSC_VER
+ int(*scin)(const void*, const void*) = linesortcompin;
+ int(*scout)(const void*, const void*) = linesortcompout;
+#else
int(*scin)() = linesortcompin;
int(*scout)() = linesortcompout;
+#endif // _MSC_VER
+
fsm = net->states;
for (i=0, numlines = 0, lasthead = 0 ; (fsm+i)->state_no != -1; i++) {
if ((fsm+i)->state_no != (fsm+i+1)->state_no || (fsm+i)->target == -1) {
diff --git a/back-ends/openfst/src/include/fst/rmfinalepsilon.h b/back-ends/openfst/src/include/fst/rmfinalepsilon.h
index 7af3d63..93e4c60 100644
--- a/back-ends/openfst/src/include/fst/rmfinalepsilon.h
+++ b/back-ends/openfst/src/include/fst/rmfinalepsilon.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_RMFINALEPSILON_H__
#define FST_LIB_RMFINALEPSILON_H__
-#ifdef USE_TR1_UNORDERED_SET
+/*#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
@@ -29,7 +29,7 @@ using std::tr1::unordered_multiset;
#include <unordered_set>
using std::unordered_set;
using std::unordered_multiset;
-#endif
+#endif*/
#include <vector>
using std::vector;
diff --git a/back-ends/openfst/src/include/fst/symbol-table-ops.h b/back-ends/openfst/src/include/fst/symbol-table-ops.h
index 34e5dcd..825edb3 100644
--- a/back-ends/openfst/src/include/fst/symbol-table-ops.h
+++ b/back-ends/openfst/src/include/fst/symbol-table-ops.h
@@ -21,7 +21,7 @@
using std::vector;
#include <string>
-#ifdef USE_TR1_UNORDERED_SET
+/*#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
@@ -29,7 +29,7 @@ using std::tr1::unordered_multiset;
#include <unordered_set>
using std::unordered_set;
using std::unordered_multiset;
-#endif
+#endif*/
#include <fst/fst.h>
#include <fst/symbol-table.h>
diff --git a/back-ends/openfst/src/include/fst/synchronize.h b/back-ends/openfst/src/include/fst/synchronize.h
index 1ae087d..bd15bb6 100644
--- a/back-ends/openfst/src/include/fst/synchronize.h
+++ b/back-ends/openfst/src/include/fst/synchronize.h
@@ -23,7 +23,7 @@
#include <algorithm>
-#ifdef USE_TR1_UNORDERED_MAP
+/*#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
@@ -41,7 +41,7 @@ using std::tr1::unordered_multiset;
#include <unordered_set>
using std::unordered_set;
using std::unordered_multiset;
-#endif
+#endif*/
#include <string>
diff --git a/back-ends/openfstwin/src/include/fst/compat.h b/back-ends/openfstwin/src/include/fst/compat.h
index b5ef862..946cc92 100644
--- a/back-ends/openfstwin/src/include/fst/compat.h
+++ b/back-ends/openfstwin/src/include/fst/compat.h
@@ -25,24 +25,30 @@
typedef SSIZE_T ssize_t;
#define snprintf _snprintf
#define strtoll _strtoi64
+
#ifndef OPENFSTEXPORT
- #ifdef _DEBUG
+#ifdef PRAGMA
+ #ifdef _DEBUG
#ifdef _M_X64
- #pragma comment (lib, "openfst64-gd.lib")
+ #pragma comment (lib, "openfst64-gd.lib")
#else
#pragma comment (lib, "openfst-gd.lib")
#endif
- #else
+ #else
#ifdef _M_X64
- #pragma comment (lib, "openfst64.lib")
+ #pragma comment (lib, "openfst64.lib")
#else
#pragma comment (lib, "openfst.lib")
#endif
- #endif
+ #endif
+#endif
#endif
+
#else
- #include <dlfcn.h>
+#ifdef DLFCN
+ #include <dlfcn.h>
#endif
+#endif // _MSC_VER
#include <climits>
#include <cstdlib>
diff --git a/back-ends/openfstwin/src/include/fst/flags.h b/back-ends/openfstwin/src/include/fst/flags.h
index 6bd8d70..0d413df 100644
--- a/back-ends/openfstwin/src/include/fst/flags.h
+++ b/back-ends/openfstwin/src/include/fst/flags.h
@@ -41,10 +41,14 @@ using std::string;
#ifdef OPENFSTEXPORT
#define OPENFSTDLL __declspec(dllexport)
#else
- #define OPENFSTDLL __declspec(dllimport)
+ #ifdef OPENFSTIMPORT
+ #define OPENFSTDLL __declspec(dllimport)
+ #else
+ #define OPENFSTDLL
+ #endif
#endif
#else
- #define OPENFSTDLL
+ #define OPENFSTDLL
#endif
//
// FLAGS USAGE:
diff --git a/back-ends/openfstwin/src/include/fst/properties.h b/back-ends/openfstwin/src/include/fst/properties.h
index 11365f1..87b82c9 100644
--- a/back-ends/openfstwin/src/include/fst/properties.h
+++ b/back-ends/openfstwin/src/include/fst/properties.h
@@ -453,7 +453,7 @@ uint64 AddArcProperties(uint64 inprops, typename A::StateId s,
return outprops;
}
-extern const char *PropertyNames[];
+OPENFSTDLL extern const char *PropertyNames[];
} // namespace fst
diff --git a/configure.ac b/configure.ac
index f4c8c58..7875d41 100644
--- a/configure.ac
+++ b/configure.ac
@@ -19,7 +19,7 @@
HFST_NAME=hfst
HFST_MAJOR=3
HFST_MINOR=8
-HFST_EXTENSION=2
+HFST_EXTENSION=3
HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
### When the VERSION is INCREMENTED, REMEMBER to increment the LONGVERSION too.
@@ -28,10 +28,10 @@ HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
LIBHFST_NAME=hfst
LIBHFST_MAJOR=3
LIBHFST_MINOR=8
-LIBHFST_EXTENSION=2
+LIBHFST_EXTENSION=3
LIBHFST_VERSION=$LIBHFST_MAJOR.$LIBHFST_MINOR.$LIBHFST_EXTENSION
-AC_INIT([hfst], [3.8.2], [hfst-bugs at helsinki.fi], [hfst])
+AC_INIT([hfst], [3.8.3], [hfst-bugs at helsinki.fi], [hfst])
AC_CONFIG_AUX_DIR([build-aux])
AM_INIT_AUTOMAKE([-Wall std-options foreign check-news])
@@ -44,8 +44,8 @@ AC_CONFIG_HEADERS([config.h libhfst/src/hfst.hpp])
AC_SUBST([LIBHFST_MAJOR], [3])
AC_SUBST([LIBHFST_MINOR], [8])
-AC_SUBST([LIBHFST_EXTENSION], [2])
-AC_SUBST([LIBHFST_VERSION], [3.8.2])
+AC_SUBST([LIBHFST_EXTENSION], [3])
+AC_SUBST([LIBHFST_VERSION], [3.8.3])
AC_SUBST([LIBHFST_NAME], [hfst])
# long version = version vector cast in base 10000, for automatic comparisons
@@ -58,7 +58,7 @@ AC_SUBST([LIBHFST_NAME], [hfst])
AC_DEFINE([HFST_LONGVERSION], [300080002L],
[Define to hfst version vector as long in base 10000])
-AC_DEFINE([HFST_REVISION], ["$Revision: 4145 $"],
+AC_DEFINE([HFST_REVISION], ["$Revision: 4409 $"],
[Automatically substitute to configure.ac revision])
AC_DEFINE_UNQUOTED([HFST_STRING], ["$PACKAGE_STRING"],
[Define to libhfst pretty name for programs to print])
@@ -97,6 +97,21 @@ AS_IF([test "x$with_foma" != xno], [AC_DEFINE([HAVE_FOMA], [1],
[Define to compile foma support in HFST])])
AM_CONDITIONAL([WANT_FOMA], [test x$with_foma != xno])
+AC_ARG_WITH([xfsm],
+ [AS_HELP_STRING([--with-xfsm],
+ [process unweighted fsts with xfsm @<:@default=no@:>@])],
+ [],
+ [with_xfsm=no])
+AS_IF([test "x$with_xfsm" != xno], [AC_DEFINE([HAVE_XFSM], [1],
+ [Define to compile xfsm support in HFST])])
+AM_CONDITIONAL([WANT_XFSM], [test x$with_xfsm != xno])
+
+AS_IF([test "x$with_xfsm" != "xno"],
+[AC_CHECK_LIB([xfsm], [main], [], AC_MSG_ERROR([xfsm back-end requested but libxfsm not found]))])
+
+AS_IF([test "x$with_xfsm" != "xno"],
+[AC_CHECK_HEADER([xfsm/xfsm_api.h], [xfsmheader_exists=yes], AC_MSG_ERROR([xfsm back-end requested but libxfsm header not found]))])
+
### Add here your library ###
#AC_ARG_WITH([my_transducer_library],
# [AS_HELP_STRING([--with-my-transducer-library],
@@ -489,20 +504,24 @@ AC_ARG_ENABLE([xfst],
[enable_xfst=$enable_all_tools])
AM_CONDITIONAL([WANT_XFST], [test x$enable_xfst != xno])
-
# Check if zlib is available
AC_CHECK_LIB([z], [main], [zlib_exists=yes], [zlib_exists=no])
-# Check if foma wrapper is wanted and possible to generate
+# Check if foma and lexc wrappers are wanted and possible to generate
AM_CONDITIONAL([GENERATE_FOMA_WRAPPER], [test "x$enable_foma_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"])
-# The same as a variable..
+AM_CONDITIONAL([GENERATE_LEXC_WRAPPER], [test "x$enable_lexc_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"])
+# The same as variables..
AS_IF([test "x$enable_foma_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"], [generate_foma_wrapper=yes], [generate_foma_wrapper=no])
+AS_IF([test "x$enable_lexc_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" != "xno"], [generate_lexc_wrapper=yes], [generate_lexc_wrapper=no])
-# Check if we have to warn the user for missing foma wrapper because zlib is not available
-AS_IF([test "x$enable_foma_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" == "xno"], [warn_about_missing_zlib=yes], [warn_about_missing_zlib=no])
+# Check if we have to warn the user for missing foma and lexc wrappers because zlib is not available
+# AS_IF([test "x$enable_foma_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" == "xno"], [warn_about_missing_zlib_foma=yes], [warn_about_missing_zlib_foma=no])
+# AS_IF([test "x$enable_lexc_wrapper" != "xno" -a "x$with_foma" != "xno" -a "x$zlib_exists" == "xno"], [warn_about_missing_zlib_lexc=yes], [warn_about_missing_zlib_lexc=no])
# If foma wrapper is not wanted or possible to generate, set these conditionals/variables to 'no'
AM_COND_IF([GENERATE_FOMA_WRAPPER], [], [AM_CONDITIONAL([WANT_FOMA_WRAPPER], [false])])
AM_COND_IF([GENERATE_FOMA_WRAPPER], [], [enable_foma_wrapper=no])
+AM_COND_IF([GENERATE_LEXC_WRAPPER], [], [AM_CONDITIONAL([WANT_LEXC_WRAPPER], [false])])
+AM_COND_IF([GENERATE_LEXC_WRAPPER], [], [enable_lexc_wrapper=no])
# Disable stress tests by default
AC_ARG_ENABLE([stress-tests],
@@ -588,6 +607,7 @@ AC_LANG_POP
AC_LANG_PUSH([C])
AM_COND_IF([GENERATE_FOMA_WRAPPER], [AC_CHECK_LIB([z], [main], [], [AC_MSG_FAILURE([foma test failed because zlib.h was not found (--without-foma to disable)])])])
+AM_COND_IF([GENERATE_LEXC_WRAPPER], [AC_CHECK_LIB([z], [main], [], [AC_MSG_FAILURE([foma test failed because zlib.h was not found (--without-foma to disable)])])])
AM_CONDITIONAL(HAVE_NCURSES,[false])
AM_CONDITIONAL(HAVE_CURSES,[false])
@@ -626,7 +646,7 @@ AS_IF([test "x$with_readline" = "xyes"],
# Checks for header files
AC_CHECK_HEADERS([limits.h stdlib.h string.h error.h glob.h locale.h langinfo.h])
-AC_CHECK_HEADERS([libxml/tree.h libxml/mem.h])
+#AC_CHECK_HEADERS([libxml/tree.h libxml/mem.h])
AC_LANG_PUSH([C++])
# Whether std::unordered_map and std::unordered_set should be used
@@ -707,6 +727,7 @@ AC_CHECK_DECLS([program_name, program_invocation_name, program_invocation_short_
AC_CHECK_DECLS([rl_completion_suppress_append, rl_completion_matches])
AC_CHECK_FUNCS([floor strchr strdup strerror strncasecmp strcspn strtol strtoul error error_at_line strndup getline getdelim getopt_long strtod xstrdup set_program_name setprogname setlocale nl_langinfo])
+AM_CONDITIONAL([WINDOWS], [test x$version_type = xwindows])
# Checks for system services
diff --git a/libhfst/src/FormatSpecifiers.h b/libhfst/src/FormatSpecifiers.h
index 076fca9..ce18500 100644
--- a/libhfst/src/FormatSpecifiers.h
+++ b/libhfst/src/FormatSpecifiers.h
@@ -13,7 +13,7 @@
#ifndef _FORMAT_SPECIFIERS_H_
#define _FORMAT_SPECIFIERS_H_
-#ifdef WINDOWS
+#if defined (_MSC_VER) || (__MINGW32__)
#define SIZE_T_SPECIFIER "%Iu"
#define SSIZE_T_SPECIFIER "%Id"
#define PTRDIFF_T_SPECIFIER "%Id"
diff --git a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
index c666cde..93ba485 100644
--- a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
+++ b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
@@ -17,7 +17,7 @@ const char * HarmonizeUnknownAndIdentitySymbols::unknown =
// --- these functions could be useful elsewhere, too
// maybe they should be in separate h- and cc-files?
-size_t max(size_t t1,size_t t2)
+size_t max_(size_t t1,size_t t2)
{ return t1 < t2 ? t2 : t1; }
static bool is_subset(const StringSet &subset,const StringSet &superset)
@@ -66,7 +66,7 @@ HarmonizeUnknownAndIdentitySymbols::HarmonizeUnknownAndIdentitySymbols
// --- typedef StringVector (not in HfstSymbolDefs.h, maybe it should be?)
std::vector<std::string> diff_vector
- (max(t1_symbol_set.size(),t2_symbol_set.size()),"");
+ (max_(t1_symbol_set.size(),t2_symbol_set.size()),"");
if (debug_harmonize)
{ debug_harmonize_print("Computing t1 symbols - t2 symbols."); }
@@ -260,7 +260,7 @@ void HarmonizeUnknownAndIdentitySymbols::harmonize_unknown_symbols
jt->get_weight())); }
}
- if (jt->get_input_symbol() == unknown and
+ if (jt->get_input_symbol() == unknown &&
jt->get_output_symbol() == unknown)
{
for (StringSet::const_iterator kt = missing_symbols.begin();
diff --git a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h
index 8a90d3e..d2fd85a 100644
--- a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h
+++ b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.h
@@ -30,6 +30,8 @@ using implementations::HfstBasicTransition;
#define debug_harmonize 0
#endif // TEST_HARMONIZE_UNKNOWN_AND_IDENTITY_SYMBOLS
+#include "hfstdll.h"
+
// --- a short documentation
class HarmonizeUnknownAndIdentitySymbols
{
@@ -42,7 +44,7 @@ class HarmonizeUnknownAndIdentitySymbols
// Constructor whose side effect it is to harmonize the identity and unknown
// symbols of its arguments.
- HarmonizeUnknownAndIdentitySymbols
+ HFSTDLL HarmonizeUnknownAndIdentitySymbols
(HfstBasicTransducer &,HfstBasicTransducer &);
protected:
@@ -56,15 +58,15 @@ class HarmonizeUnknownAndIdentitySymbols
// of HfstBasicTransducers can sometimes unexplainedly be empty...
// --- the alphabet can contain also symbols that are not found in
// transitions...
- void populate_symbol_set(const HfstBasicTransducer &,StringSet &);
+ HFSTDLL void populate_symbol_set(const HfstBasicTransducer &,StringSet &);
// Add all symbols in the StringSet to the alphabet of the transducer.
- void add_symbols_to_alphabet(HfstBasicTransducer &, const StringSet &);
+ HFSTDLL void add_symbols_to_alphabet(HfstBasicTransducer &, const StringSet &);
// For every x in the set, add x:x transitions for every identity:identity
// transition in the argument transducer (the source and target states as
// well as the weights are the same as in the original identity transition.
- void harmonize_identity_symbols
+ HFSTDLL void harmonize_identity_symbols
(HfstBasicTransducer &,const StringSet &);
// For every x in the set
@@ -78,13 +80,14 @@ class HarmonizeUnknownAndIdentitySymbols
//
// (the source and target states as well as the weights are the same as in
// the original identity transition)
- void harmonize_unknown_symbols
+ HFSTDLL void harmonize_unknown_symbols
(HfstBasicTransducer &,const StringSet &);
};
-void debug_harmonize_print(const StringSet &);
-void debug_harmonize_print(const std::string &);
-size_t max(size_t t1,size_t t2); // --- a short documentation
+HFSTDLL void debug_harmonize_print(const StringSet &);
+HFSTDLL void debug_harmonize_print(const std::string &);
+// use name 'max_' to avoid collision with windows macro 'max'
+HFSTDLL size_t max_(size_t t1,size_t t2); // --- a short documentation
}
diff --git a/libhfst/src/HfstApply.cc b/libhfst/src/HfstApply.cc
index 332b307..7b333ff 100644
--- a/libhfst/src/HfstApply.cc
+++ b/libhfst/src/HfstApply.cc
@@ -39,6 +39,9 @@ namespace hfst
if (converted == FOMA_TYPE) {
return false;
}
+ if (converted == XFSM_TYPE) {
+ return false;
+ }
}
/* Add here your library. */
//#ifdef HAVE_MY_TRANSDUCER_LIBRARY
@@ -73,6 +76,9 @@ namespace hfst
#if HAVE_FOMA
fsm * (*foma_funct)(fsm *),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr),
+#endif
/* Add your library. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)(my_namespace::MyFst *),
@@ -122,6 +128,13 @@ namespace hfst
break;
}
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ {
+ (void)xfsm_funct(implementation.xfsm);
+ break;
+ }
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
@@ -156,6 +169,9 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
#if HAVE_FOMA
fsm * (*foma_funct)(fsm *, unsigned int n),
#endif
+#if HAVE_XFSM
+NETptr (*xfsm_funct)(NETptr, unsigned int n),
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
// my_namespace::MyFst *
@@ -205,6 +221,16 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
break;
}
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ {
+ NETptr xfsm_temp =
+ xfsm_funct(implementation.xfsm,n);
+ delete implementation.xfsm;
+ implementation.xfsm = xfsm_temp;
+ break;
+ }
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
@@ -239,6 +265,9 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
#if HAVE_FOMA
fsm * (*foma_funct)(fsm *, String, String),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr, String, String),
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst *
@@ -288,6 +317,16 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
break;
}
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ {
+ NETptr xfsm_temp =
+ xfsm_funct(implementation.xfsm,s1,s2);
+ delete implementation.xfsm;
+ implementation.xfsm = xfsm_temp;
+ break;
+ }
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
@@ -325,6 +364,9 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
fsm * (*foma_funct)(fsm *,
fsm *),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr, NETptr),
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)(my_namespace::MyFst *,
@@ -347,7 +389,6 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
/* special symbols are never harmonized */
this->insert_missing_symbols_to_alphabet_from(another, true);
another.insert_missing_symbols_to_alphabet_from(*this, true);
-
HfstTransducer * another_ =
this->harmonize_(another);
if (another_ == NULL) // foma
@@ -398,6 +439,16 @@ SFST::Transducer * (*sfst_funct)(SFST::Transducer *, unsigned int n),
break;
}
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ {
+ NETptr xfsm_temp =
+ xfsm_funct(implementation.xfsm,another_->implementation.xfsm);
+ //delete implementation.xfsm;
+ implementation.xfsm = xfsm_temp;
+ break;
+ }
+#endif
/* Add your library here. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
diff --git a/libhfst/src/HfstDataTypes.cc b/libhfst/src/HfstDataTypes.cc
index 99ccd7b..f716889 100644
--- a/libhfst/src/HfstDataTypes.cc
+++ b/libhfst/src/HfstDataTypes.cc
@@ -19,6 +19,9 @@ namespace hfst
case FOMA_TYPE:
return "FOMA_TYPE";
break;
+ case XFSM_TYPE:
+ return "XFSM_TYPE";
+ break;
case HFST_OL_TYPE:
return "HFST_OL_TYPE";
break;
@@ -56,6 +59,9 @@ namespace hfst
case FOMA_TYPE:
return "foma";
break;
+ case XFSM_TYPE:
+ return "xfsm";
+ break;
case HFST_OL_TYPE:
return "hfst-optimized-lookup-unweighted";
break;
diff --git a/libhfst/src/HfstDataTypes.h b/libhfst/src/HfstDataTypes.h
index a3d028e..68908fc 100644
--- a/libhfst/src/HfstDataTypes.h
+++ b/libhfst/src/HfstDataTypes.h
@@ -46,6 +46,7 @@ namespace hfst
TROPICAL_OPENFST_TYPE, /**< An OpenFst transducer with tropical weights. */
LOG_OPENFST_TYPE, /**< An OpenFst transducer with logarithmic weights. */
FOMA_TYPE, /**< A foma transducer, unweighted. */
+ XFSM_TYPE, /**< An xfsm transducer, unweighted (mostly for testing purposes). */
/* Add an enumerator for your transducer type here. */
//MY_TRANSDUCER_LIBRARY_TYPE,
HFST_OL_TYPE, /**< An HFST optimized lookup transducer, unweighted */
diff --git a/libhfst/src/HfstEpsilonHandler.cc b/libhfst/src/HfstEpsilonHandler.cc
index 1c1f1b4..512ee4c 100644
--- a/libhfst/src/HfstEpsilonHandler.cc
+++ b/libhfst/src/HfstEpsilonHandler.cc
@@ -12,7 +12,7 @@ namespace hfst {
Appends state \a s to the epsilon path if not found at the end already. */
void HfstEpsilonHandler::push_back(hfst::implementations::HfstState s)
{
- if (not epsilon_path.empty()) {
+ if (! epsilon_path.empty()) {
if (epsilon_path.back() != s ) {
epsilon_path.push_back(s);
}
@@ -26,7 +26,7 @@ namespace hfst {
unless the path it is empty. */
void HfstEpsilonHandler::pop_back()
{
- if (not epsilon_path.empty())
+ if (! epsilon_path.empty())
epsilon_path.pop_back();
};
diff --git a/libhfst/src/HfstExceptionDefs.h b/libhfst/src/HfstExceptionDefs.h
index b461334..7df8fe1 100644
--- a/libhfst/src/HfstExceptionDefs.h
+++ b/libhfst/src/HfstExceptionDefs.h
@@ -7,6 +7,8 @@
void hfst_set_exception(std::string name);
std::string hfst_get_exception();
+#include "hfstdll.h"
+
//! @file HfstExceptionDefs.h
//! @brief A file for exceptions
@@ -20,7 +22,7 @@ struct HfstException
HfstException(void);
HfstException(const std::string &name,const std::string &file,size_t line);
//! @brief Get the error message.
- std::string operator() (void) const;
+ HFSTDLL std::string operator() (void) const;
};
//! @brief Macro to throw an exception of type @a E.
diff --git a/libhfst/src/HfstFlagDiacritics.cc b/libhfst/src/HfstFlagDiacritics.cc
index 07b9070..f9257d1 100644
--- a/libhfst/src/HfstFlagDiacritics.cc
+++ b/libhfst/src/HfstFlagDiacritics.cc
@@ -35,8 +35,8 @@ bool FdOperation::is_diacritic(const std::string& diacritic_string)
}
if (diacritic_string.find_last_of('.') == 2)
{
- if ((diacritic_string.at(1) != 'R') and
- (diacritic_string.at(1) != 'D') and
+ if ((diacritic_string.at(1) != 'R') &&
+ (diacritic_string.at(1) != 'D') &&
(diacritic_string.at(1) != 'C'))
{ return false; }
}
@@ -57,13 +57,25 @@ bool FdOperation::is_diacritic(const std::string& diacritic_string)
// The feature name ends at the '.' char after the feature name star pos.
size_t feature_past = diacritic.find('.',3);
+ // If there is no value given (e.g. "@D.FOO@"),
+ if (feature_past == std::string::npos)
+ feature_past = diacritic.size() - 1; // point to the last '@' char.
+
return diacritic.substr(feature_start, feature_past - feature_start);
}
std::string FdOperation::get_value(const std::string& diacritic)
{
+ // First locate the second '.' char.
+ size_t second_comma = diacritic.find('.',diacritic.find('.') + 1);
+
+ // If there is no second '.' char (diacritics is e.g. of form "@D.FOO@"),
+ // return an empty string.
+ if (second_comma == std::string::npos)
+ return "";
+
// The value starts after the second '.' char.
- size_t value_start = diacritic.find('.',diacritic.find('.') + 1) + 1;
+ size_t value_start = second_comma + 1;
// The value ends at the last char.
size_t value_past = diacritic.size() - 1;
diff --git a/libhfst/src/HfstFlagDiacritics.h b/libhfst/src/HfstFlagDiacritics.h
index d342812..ff9d7bd 100644
--- a/libhfst/src/HfstFlagDiacritics.h
+++ b/libhfst/src/HfstFlagDiacritics.h
@@ -8,6 +8,8 @@
#include <cassert>
#include <utility>
+#include "hfstdll.h"
+
/** @file HfstFlagDiacritics.h
\brief Class declarations for flag diacritic handling. */
@@ -26,16 +28,16 @@ private:
FdValue value;
std::string name;
public:
- FdOperation
+ HFSTDLL FdOperation
(FdOperator op, FdFeature feat, FdValue val, const std::string& str):
op(op), feature(feat), value(val), name(str) {}
- FdOperator Operator(void) const { return op; }
- FdFeature Feature(void) const { return feature; }
- FdValue Value(void) const { return value; }
- std::string Name(void) const { return name; }
+ HFSTDLL FdOperator Operator(void) const { return op; }
+ HFSTDLL FdFeature Feature(void) const { return feature; }
+ HFSTDLL FdValue Value(void) const { return value; }
+ HFSTDLL std::string Name(void) const { return name; }
- static FdOperator char_to_operator(char c)
+ HFSTDLL static FdOperator char_to_operator(char c)
{
switch (c) {
case 'P': return Pop;
@@ -49,15 +51,15 @@ public:
}
}
- static bool is_diacritic(const std::string& diacritic_str);
- static std::string::size_type find_diacritic
+ HFSTDLL static bool is_diacritic(const std::string& diacritic_str);
+ HFSTDLL static std::string::size_type find_diacritic
(const std::string& diacritic_str,
std::string::size_type& length);
- static std::string get_operator(const std::string& diacritic);
- static std::string get_feature(const std::string& diacritic);
- static std::string get_value(const std::string& diacritic);
- static bool has_value(const std::string& diacritic);
+ HFSTDLL static std::string get_operator(const std::string& diacritic);
+ HFSTDLL static std::string get_feature(const std::string& diacritic);
+ HFSTDLL static std::string get_value(const std::string& diacritic);
+ HFSTDLL static bool has_value(const std::string& diacritic);
};
template<class T> class FdState;
diff --git a/libhfst/src/HfstInputStream.cc b/libhfst/src/HfstInputStream.cc
index 86455ec..57cdd69 100644
--- a/libhfst/src/HfstInputStream.cc
+++ b/libhfst/src/HfstInputStream.cc
@@ -116,6 +116,7 @@ namespace hfst
assert(false);
break;
}
+ HFST_THROW(HfstFatalException); // make compiler happy
}
short &HfstInputStream::stream_get(short &i)
@@ -162,6 +163,7 @@ namespace hfst
assert(false);
break;
}
+ HFST_THROW(HfstFatalException); // make compiler happy
}
unsigned short &HfstInputStream::stream_get(unsigned short &i)
@@ -212,6 +214,7 @@ namespace hfst
assert(false);
break;
}
+ HFST_THROW(HfstFatalException); // make compiler happy
}
void HfstInputStream::stream_unget(char c)
@@ -305,25 +308,28 @@ namespace hfst
void HfstInputStream::read_transducer(HfstTransducer &t)
{
- if (input_stream != NULL) { // first transducer in the stream
- input_stream = NULL;
- if (stream_eof())
- HFST_THROW(EndOfStreamException);
- // if header bytes have been read from a file, skip these bytes
- if (strcmp(filename.c_str(), "") != 0)
- ignore(bytes_to_skip);
- }
- else {
- if (stream_eof())
- HFST_THROW(EndOfStreamException);
- ImplementationType current_type = this->get_type();
- ImplementationType stype = this->stream_fst_type();
- if (stype != current_type) {
- HFST_THROW_MESSAGE(TransducerTypeMismatchException,
- "HfstInputStream contains HfstTransducers "
- "whose type is not the same");
+ if (type != XFSM_TYPE)
+ {
+ if (input_stream != NULL) { // first transducer in the stream
+ input_stream = NULL;
+ if (stream_eof())
+ HFST_THROW(EndOfStreamException);
+ // if header bytes have been read from a file, skip these bytes
+ if (strcmp(filename.c_str(), "") != 0)
+ ignore(bytes_to_skip);
+ }
+ else {
+ if (stream_eof())
+ HFST_THROW(EndOfStreamException);
+ ImplementationType current_type = this->get_type();
+ ImplementationType stype = this->stream_fst_type();
+ if (stype != current_type) {
+ HFST_THROW_MESSAGE(TransducerTypeMismatchException,
+ "HfstInputStream contains HfstTransducers "
+ "whose type is not the same");
+ }
+ }
}
- }
switch (type)
{
@@ -365,7 +371,7 @@ namespace hfst
and/or that epsilon is coded differently than "@_EPSILON_SYMBOL_@"
and/or that numbers 1 and 2 are reserved for other use than
"@_UNKNOWN_SYMBOL_@" or "@_IDENTITY_SYMBOL_@". */
- if (not has_hfst_header)
+ if (! has_hfst_header)
{
HfstBasicTransducer * net =
ConversionFunctions::
@@ -481,7 +487,7 @@ namespace hfst
and/or that epsilon is coded differently than "@_EPSILON_SYMBOL_@"
and/or that numbers 1 and 2 are reserved for other use than
"@_UNKNOWN_SYMBOL_@" or "@_IDENTITY_SYMBOL_@". */
- if (not has_hfst_header)
+ if (! has_hfst_header)
{
HfstBasicTransducer * net =
ConversionFunctions::
@@ -509,6 +515,12 @@ namespace hfst
this->implementation.foma->read_transducer();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ t.implementation.xfsm =
+ this->implementation.xfsm->read_transducer();
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
t.implementation.my_transducer_library =
@@ -529,13 +541,16 @@ namespace hfst
break;
}
- t.set_name(name);
- for (std::map<string,string>::const_iterator prop = props.begin();
- prop != props.end();
- ++prop)
- {
- t.set_property(prop->first, prop->second);
- }
+ if (type != XFSM_TYPE)
+ {
+ t.set_name(name);
+ for (std::map<string,string>::const_iterator prop = props.begin();
+ prop != props.end();
+ ++prop)
+ {
+ t.set_property(prop->first, prop->second);
+ }
+ }
}
@@ -619,7 +634,12 @@ namespace hfst
break;
}
default:
+#ifdef HAVE_XFSM
+ return XFSM_;
+#else
return ERROR_TYPE_;
+#endif
+
}
return ERROR_TYPE_;
}
@@ -633,7 +653,7 @@ namespace hfst
}
// (1) first pair "version", "3.0"
- if ( not ( ( strcmp("version", header_data[0].first.c_str()) == 0 ) &&
+ if ( ! ( ( strcmp("version", header_data[0].first.c_str()) == 0 ) &&
( ( strcmp("3.0", header_data[0].second.c_str()) == 0 ) ||
( strcmp("3.3", header_data[0].second.c_str()) == 0 ) ) ) ){
HFST_THROW_MESSAGE(TransducerHeaderException,
@@ -641,7 +661,7 @@ namespace hfst
}
// (2) second pair "type", (valid type field)
- if ( not ( strcmp("type", header_data[1].first.c_str()) == 0 ) ) {
+ if ( ! ( strcmp("type", header_data[1].first.c_str()) == 0 ) ) {
HFST_THROW_MESSAGE(TransducerHeaderException,
"Hfst header: transducer type not given");
}
@@ -857,7 +877,7 @@ namespace hfst
}
// whether the stream contains an HFST version <3.0 transducer
- // or an SFST, OpenFst or a foma transducer
+ // or a native SFST, OpenFst, foma or xfsm transducer
TransducerType transducer_type = guess_fst_type(bytes_read);
bytes_to_skip=bytes_read;
@@ -888,6 +908,9 @@ namespace hfst
case FOMA_:
return FOMA_TYPE;
break;
+ case XFSM_:
+ return XFSM_TYPE;
+ break;
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_:
return MY_TRANSDUCER_LIBRARY_TYPE;
@@ -916,7 +939,7 @@ namespace hfst
catch (const HfstException e)
{ throw e; }
- if ( not HfstTransducer::is_implementation_type_available(type)) {
+ if ( ! HfstTransducer::is_implementation_type_available(type)) {
HFST_THROW(ImplementationTypeNotAvailableException);
}
@@ -944,6 +967,12 @@ namespace hfst
implementation.foma = new hfst::implementations::FomaInputStream;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ // Will throw an error.
+ implementation.xfsm = new hfst::implementations::XfsmInputStream;
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library
@@ -992,7 +1021,7 @@ namespace hfst
catch( const HfstException e)
{ throw e; }
- if ( not HfstTransducer::is_implementation_type_available(type)) {
+ if ( ! HfstTransducer::is_implementation_type_available(type)) {
HFST_THROW(ImplementationTypeNotAvailableException);
}
@@ -1028,6 +1057,12 @@ namespace hfst
= new hfst::implementations::FomaInputStream(filename);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm
+ = new hfst::implementations::XfsmInputStream(filename);
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library
@@ -1074,6 +1109,11 @@ namespace hfst
delete implementation.foma;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ delete implementation.xfsm;
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
delete implementation.my_transducer_library;
@@ -1114,6 +1154,11 @@ namespace hfst
implementation.foma->close();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm->close();
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library->close();
@@ -1152,6 +1197,11 @@ namespace hfst
return implementation.foma->is_eof();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ return implementation.xfsm->is_eof();
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
return implementation.my_transducer_library->is_eof();
@@ -1190,6 +1240,11 @@ namespace hfst
return implementation.foma->is_bad();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ return implementation.xfsm->is_bad();
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
return implementation.my_transducer_library->is_bad();
@@ -1229,6 +1284,11 @@ namespace hfst
return implementation.foma->is_good();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ return implementation.xfsm->is_good();
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
return implementation.my_transducer_library->is_good();
diff --git a/libhfst/src/HfstInputStream.h b/libhfst/src/HfstInputStream.h
index 106255e..6b4e03e 100644
--- a/libhfst/src/HfstInputStream.h
+++ b/libhfst/src/HfstInputStream.h
@@ -18,6 +18,8 @@
#include "HfstDataTypes.h"
+#include "hfstdll.h"
+
/** @file HfstInputStream.h
\brief Declaration of class HfstInputStream.
*/
@@ -38,6 +40,9 @@ namespace hfst
#if HAVE_FOMA
class FomaInputStream;
#endif
+#if HAVE_XFSM
+ class XfsmInputStream;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
class MyTransducerLibraryInputStream;
#endif
@@ -109,6 +114,9 @@ For documentation on the HFST binary transducer format, see
#if HAVE_FOMA
hfst::implementations::FomaInputStream * foma;
#endif
+#if HAVE_XFSM
+ hfst::implementations::XfsmInputStream * xfsm;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
hfst::implementations::MyTransducerLibraryInputStream *
@@ -187,6 +195,8 @@ For documentation on the HFST binary transducer format, see
SFST_,
/* A foma transducer. */
FOMA_,
+ /* An xfsm transducer. */
+ XFSM_,
/* Your transducer type */
//MY_TRANSDUCER_LIBRARY_,
/* Transducer type not recognized. */
@@ -223,7 +233,7 @@ For documentation on the HFST binary transducer format, see
@throws EndOfStreamException
@throws TransducerHeaderException
*/
- HfstInputStream(void);
+ HFSTDLL HfstInputStream(void);
/** \brief Open a stream to file \a filename for reading binary
transducers.
@@ -236,29 +246,29 @@ For documentation on the HFST binary transducer format, see
@throws EndOfStreamException
@throws TransducerHeaderException
*/
- HfstInputStream(const std::string &filename);
+ HFSTDLL HfstInputStream(const std::string &filename);
/** \brief Destructor. */
- ~HfstInputStream(void);
+ HFSTDLL ~HfstInputStream(void);
/** \brief Close the stream.
If the stream points to standard input, nothing is done. */
- void close(void);
+ HFSTDLL void close(void);
/** \brief Whether the stream is at end. */
- bool is_eof(void);
+ HFSTDLL bool is_eof(void);
/** \brief Whether badbit is set. */
- bool is_bad(void);
+ HFSTDLL bool is_bad(void);
/** \brief Whether the state of the stream is good for input operations. */
- bool is_good(void);
+ HFSTDLL bool is_good(void);
/** \brief The type of the first transducer in the stream.
By default, all transducers in a stream have the same type, else
a TransducerTypeMismatchException is thrown when reading the first
transducer that has a different type than the previous ones. */
- ImplementationType get_type(void) const;
+ HFSTDLL ImplementationType get_type(void) const;
friend class HfstTransducer;
};
diff --git a/libhfst/src/HfstLookupFlagDiacritics.cc b/libhfst/src/HfstLookupFlagDiacritics.cc
index b4aa7a9..6ce6c04 100644
--- a/libhfst/src/HfstLookupFlagDiacritics.cc
+++ b/libhfst/src/HfstLookupFlagDiacritics.cc
@@ -37,8 +37,8 @@ bool FlagDiacriticTable::is_genuine_diacritic
}
if (diacritic_string.find_last_of('.') == 2)
{
- if ((diacritic_string.at(1) != 'R') and
- (diacritic_string.at(1) != 'D') and
+ if ((diacritic_string.at(1) != 'R') &&
+ (diacritic_string.at(1) != 'D') &&
(diacritic_string.at(1) != 'C'))
{ return false; }
}
@@ -80,8 +80,8 @@ void FlagDiacriticTable::split_diacritic(const std::string &diacritic_string)
size_t last_char_pos = diacritic_string.size() - 1;
if (second_full_stop_pos == std::string::npos)
{
- assert((diacritic_operators[diacritic_string] == Cop) or
- (diacritic_operators[diacritic_string] == Dop) or
+ assert((diacritic_operators[diacritic_string] == Cop) ||
+ (diacritic_operators[diacritic_string] == Dop) ||
(diacritic_operators[diacritic_string] == Rop));
diacritic_has_value[diacritic_string] = false;
diacritic_features[diacritic_string] =
@@ -130,7 +130,7 @@ void FlagDiacriticTable::disallow(std::string &feature,
if (feature_values.find(feature) == feature_values.end())
{ return; }
if (feature_values[feature] == value)
- { error_flag = error_flag or feature_polarities[feature]; }
+ { error_flag = error_flag || feature_polarities[feature]; }
}
void FlagDiacriticTable::disallow(std::string &feature)
{
@@ -148,7 +148,7 @@ void FlagDiacriticTable::require(std::string &feature,
else if (feature_values[feature] != value)
{ error_flag = true; }
else
- { error_flag = error_flag or (not feature_polarities[feature]); }
+ { error_flag = error_flag || (! feature_polarities[feature]); }
}
void FlagDiacriticTable::require(std::string &feature)
{
@@ -164,7 +164,7 @@ void FlagDiacriticTable::unify(std::string &feature,
// If feature set to something else negatively, set it to value.
else if (feature_values[feature] != value)
{
- if (not feature_polarities[feature])
+ if (! feature_polarities[feature])
{ set_positive_value(feature,value); }
}
require(feature,value);
@@ -196,7 +196,7 @@ void FlagDiacriticTable::insert_symbol(const std::string &symbol)
diacritic_values[symbol]);
break;
case Dop:
- if (not diacritic_has_value[symbol])
+ if (! diacritic_has_value[symbol])
{
disallow(diacritic_features[symbol]);
}
@@ -207,7 +207,7 @@ void FlagDiacriticTable::insert_symbol(const std::string &symbol)
}
break;
case Rop:
- if (not diacritic_has_value[symbol])
+ if (! diacritic_has_value[symbol])
{
require(diacritic_features[symbol]);
}
@@ -262,7 +262,7 @@ StringVector FlagDiacriticTable::filter_diacritics
it != input_string.end();
++it)
{
- if (not is_diacritic(*it))
+ if (! is_diacritic(*it))
{ filtered.push_back(*it); }
}
return filtered;
@@ -313,54 +313,54 @@ int main(void)
FlagDiacriticTable fdt;
std::cout << "\"\" should pass:";
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"a\" should pass:";
fdt.insert_number(100);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@P.NeedNoun.ON@ @R.NeedNoun.ON@\" should pass:";
fdt.insert_number(1);
fdt.insert_number(3);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@P.NeedNoun.ON@ @D.NeedNoun.ON@\" should fail:";
fdt.insert_number(1);
fdt.insert_number(4);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == true);
fdt.reset();
std::cout << "\"@P.NeedNoun.ON@ a @R.NeedNoun.ON@\" should pass:";
fdt.insert_number(1);
fdt.insert_number(100);
fdt.insert_number(3);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@N.NeedNoun.ON@ @D.NeedNoun.ON@\" should pass:";
fdt.insert_number(2);
fdt.insert_number(4);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@D.NeedNoun.ON@\" should pass:";
fdt.insert_number(4);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@R.NeedNoun.ON@\" should fail:";
fdt.insert_number(3);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == true);
fdt.reset();
std::cout << "\"@P.NeedNoun.ON@ @C.NeedNoun@ @D.NeedNoun.ON@\" should pass:";
fdt.insert_number(1);
fdt.insert_number(6);
fdt.insert_number(4);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout
@@ -370,7 +370,7 @@ int main(void)
fdt.insert_number(7);
fdt.insert_number(6);
fdt.insert_number(4);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout
@@ -381,7 +381,7 @@ int main(void)
fdt.insert_number(6);
fdt.insert_number(4);
fdt.insert_number(8);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout
@@ -391,30 +391,30 @@ int main(void)
fdt.insert_number(6);
fdt.insert_number(4);
fdt.insert_number(8);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == true);
fdt.reset();
std::cout << "\"@U.NeedNoun.ON@\" should pass:";
fdt.insert_number(5);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@P.NeedNoun.ON@ @U.NeedNoun.ON@\" should pass:";
fdt.insert_number(1);
fdt.insert_number(5);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@N.NeedNoun.ON@ @U.NeedNoun.ON@\" should fail:";
fdt.insert_number(2);
fdt.insert_number(5);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == true);
fdt.reset();
std::cout << "\"@N.NeedNoun.foo@ @U.NeedNoun.ON@\" should pass:";
fdt.insert_number(9);
fdt.insert_number(5);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
std::cout << "\"@N.NeedNoun.ON@ @C.NeedNoun@ @U.NeedNoun.ON@\""
@@ -422,7 +422,7 @@ int main(void)
fdt.insert_number(2);
fdt.insert_number(6);
fdt.insert_number(5);
- std::cout << " " << not fdt.fails() << std::endl;
+ std::cout << " " << ! fdt.fails() << std::endl;
assert(fdt.fails() == false);
fdt.reset();
short kv[3] = {2,6,5};
diff --git a/libhfst/src/HfstOutputStream.cc b/libhfst/src/HfstOutputStream.cc
index e22ef82..313ae92 100644
--- a/libhfst/src/HfstOutputStream.cc
+++ b/libhfst/src/HfstOutputStream.cc
@@ -24,7 +24,7 @@ namespace hfst
HfstOutputStream::HfstOutputStream(ImplementationType type, bool hfst_format):
type(type), hfst_format(hfst_format), is_open(false)
{
- if (not HfstTransducer::is_implementation_type_available(type)) {
+ if (! HfstTransducer::is_implementation_type_available(type)) {
HFST_THROW(ImplementationTypeNotAvailableException);
}
@@ -54,6 +54,12 @@ namespace hfst
new hfst::implementations::FomaOutputStream();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ new hfst::implementations::XfsmOutputStream(); // throws error, not implemented
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library =
@@ -74,13 +80,14 @@ namespace hfst
}
this->is_open=true;
}
+
// FIXME: HfstOutputStream takes a string parameter,
// HfstInputStream a const char*
HfstOutputStream::HfstOutputStream
- (const std::string &filename,ImplementationType type, bool hfst_format):
- type(type), hfst_format(hfst_format), is_open(false)
+ (const std::string &filename,ImplementationType type, bool hfst_format_):
+ type(type), hfst_format(hfst_format_), is_open(false)
{
- if (not HfstTransducer::is_implementation_type_available(type)) {
+ if (! HfstTransducer::is_implementation_type_available(type)) {
HFST_THROW(ImplementationTypeNotAvailableException);
}
@@ -116,6 +123,15 @@ namespace hfst
new hfst::implementations::FomaOutputStream(filename);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ /* XFSM api only offers a function that reads transducers that takes a filename argument.
+ That is why we don't write an HFST header. */
+ hfst_format = false;
+ implementation.xfsm =
+ new hfst::implementations::XfsmOutputStream(filename);
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library =
@@ -161,6 +177,11 @@ namespace hfst
delete implementation.foma;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ delete implementation.xfsm;
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
delete implementation.my_transducer_library;
@@ -218,6 +239,11 @@ namespace hfst
implementation.foma->write(c);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ throw "operation XfsmOutputStream::write(const char &c) not supported";
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library->write(c);
@@ -264,6 +290,11 @@ namespace hfst
type_value=std::string("FOMA");
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ type_value=std::string("XFSM");
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
type_value=std::string("MY_TRANSDUCER_LIBRARY");
@@ -307,6 +338,19 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
}
}
+ HfstOutputStream &HfstOutputStream::flush()
+ {
+ if (! this->is_open) {
+ HFST_THROW(StreamIsClosedException); }
+#if HAVE_XFSM
+ if (type == XFSM_TYPE)
+ {
+ implementation.xfsm->flush();
+ }
+#endif
+ return *this;
+ }
+
HfstOutputStream &HfstOutputStream::redirect (HfstTransducer &transducer)
{
return this->operator<<(transducer);
@@ -314,7 +358,7 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
HfstOutputStream &HfstOutputStream::operator<< (HfstTransducer &transducer)
{
- if (not this->is_open) {
+ if (! this->is_open) {
HFST_THROW(StreamIsClosedException);
}
@@ -355,8 +399,12 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
HFST version 3.0 header must contain at least the attributes 'version',
'type' and 'name' and their values. Implementation-specific attributes
can follow after these obligatory attributes.
+
+ Note: in XFSM format, we never write the HFST header. hfst_format is always
+ false if the stream is of XFSM format.
*/
if (hfst_format) {
+
const int MAX_HEADER_LENGTH=65535;
// collect the header data here
@@ -425,6 +473,13 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
(transducer.implementation.foma);
return *this;
#endif
+#if HAVE_XFSM
+ /* This stores the transducer in a list that is written only when flush() is called. */
+ case XFSM_TYPE:
+ implementation.xfsm->write_transducer
+ (transducer.implementation.xfsm);
+ return *this;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library->write_transducer
@@ -464,6 +519,11 @@ HfstOutputStream::append_implementation_specific_header_data(std::vector<char>&,
implementation.foma->close();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm->close();
+ break;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
case MY_TRANSDUCER_LIBRARY_TYPE:
implementation.my_transducer_library->close();
diff --git a/libhfst/src/HfstOutputStream.h b/libhfst/src/HfstOutputStream.h
index db816a7..c368e3d 100644
--- a/libhfst/src/HfstOutputStream.h
+++ b/libhfst/src/HfstOutputStream.h
@@ -18,6 +18,8 @@
#include "HfstDataTypes.h"
+#include "hfstdll.h"
+
/** @file HfstOutputStream.h
\brief Declaration of class HfstOutputStream. */
@@ -37,6 +39,9 @@ namespace hfst
#if HAVE_FOMA
class FomaOutputStream;
#endif
+#if HAVE_XFSM
+ class XfsmOutputStream;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
class MyTransducerLibraryOutputStream;
#endif
@@ -81,6 +86,9 @@ For more information on HFST transducer structure, see
#if HAVE_FOMA
hfst::implementations::FomaOutputStream * foma;
#endif
+#if HAVE_XFSM
+ hfst::implementations::XfsmOutputStream * xfsm;
+#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
hfst::implementations::MyTransducerLibraryOutputStream *
@@ -120,7 +128,7 @@ For more information on HFST transducer structure, see
\a hfst_format defines whether transducers are written
in hfst format or as such in their backend format.
*/
- HfstOutputStream(ImplementationType type, bool hfst_format=true);
+ HFSTDLL HfstOutputStream(ImplementationType type, bool hfst_format=true);
/** \brief Open a stream to file \a filename for writing binary transducers
of type \a type.
@@ -129,10 +137,16 @@ For more information on HFST transducer structure, see
If the file exists, it is overwritten.
*/
- HfstOutputStream(const std::string &filename, ImplementationType type, bool hfst_format=true);
+ HFSTDLL HfstOutputStream(const std::string &filename, ImplementationType type, bool hfst_format=true);
/** \brief Destructor. */
- ~HfstOutputStream(void);
+ HFSTDLL ~HfstOutputStream(void);
+
+ /** \brief Flush the stream.
+
+ If the stream is of XFSM_TYPE, all transducers inserted with the operator<<
+ are actually written to the stream. Else, does nothing. */
+ HFSTDLL HfstOutputStream &flush();
/** \brief Write the transducer \a transducer in binary format
to the stream.
@@ -140,19 +154,22 @@ For more information on HFST transducer structure, see
All transducers must have the same type as the stream, else a
TransducerTypeMismatchException is thrown.
+ If the stream is of XFSM_TYPE, \a transducer is stored to a list
+ and written when flush() is called for the stream.
+
@throws TransducerTypeMismatchException
*/
- HfstOutputStream &operator<< (HfstTransducer &transducer);
+ HFSTDLL HfstOutputStream &operator<< (HfstTransducer &transducer);
/** @brief An alias for operator<<.
@see operator<< */
- HfstOutputStream& redirect (HfstTransducer &transducer);
+ HFSTDLL HfstOutputStream& redirect (HfstTransducer &transducer);
/** \brief Close the stream.
If the stream points to standard output, nothing is done. */
- void close(void);
+ HFSTDLL void close(void);
};
diff --git a/libhfst/src/HfstRules.cc b/libhfst/src/HfstRules.cc
index dc4b0a9..7ee3939 100644
--- a/libhfst/src/HfstRules.cc
+++ b/libhfst/src/HfstRules.cc
@@ -289,8 +289,8 @@ namespace hfst
HfstTransducer t2_proj(context.second);
t2_proj.input_project();
- if ( not t1_proj.compare(context.first) ||
- not t2_proj.compare(context.second) ) {
+ if ( ! t1_proj.compare(context.first) ||
+ ! t2_proj.compare(context.second) ) {
HFST_THROW(ContextTransducersAreNotAutomataException); }
std::string leftm("@_LEFT_MARKER_@");
@@ -563,7 +563,7 @@ namespace hfst
for (HfstTransducerPairVector::const_iterator it = contexts.begin();
it != contexts.end(); it++)
{
- if (not type_defined) {
+ if (! type_defined) {
type = it->first.get_type();
type_defined=true;
}
@@ -578,7 +578,7 @@ namespace hfst
"rules::restriction");
}
}
- if (not type_defined) {
+ if (! type_defined) {
HFST_THROW_MESSAGE(EmptySetOfContextsException, "rules::restriction");
}
@@ -659,9 +659,10 @@ namespace hfst
return retval1.intersect(retval2);
}
- else
+ else {
assert(false);
-
+ return HfstTransducer(type); // make compiler happy
+ }
}
HfstTransducer restriction(HfstTransducerPairVector &contexts,
diff --git a/libhfst/src/HfstSymbolDefs.cc b/libhfst/src/HfstSymbolDefs.cc
index e67b341..20fc130 100644
--- a/libhfst/src/HfstSymbolDefs.cc
+++ b/libhfst/src/HfstSymbolDefs.cc
@@ -118,6 +118,19 @@ bool is_default(const char * str)
return result;
}
+ StringVector to_string_vector(const StringPairVector & spv, bool input_side)
+ {
+ StringVector result;
+ for (StringPairVector::const_iterator it = spv.begin(); it != spv.end(); it++)
+ {
+ if (input_side)
+ result.push_back(it->first);
+ else
+ result.push_back(it->second);
+ }
+ return result;
+ }
+
StringVector to_string_vector(const HfstTwoLevelPath & path)
{
StringVector result;
@@ -205,7 +218,7 @@ bool is_default(const char * str)
it != v.end();
++it)
{
- if (not FdOperation::is_diacritic(*it))
+ if (! FdOperation::is_diacritic(*it))
{ v_wo_flags.push_back(*it); }
}
return v_wo_flags;
@@ -218,8 +231,8 @@ bool is_default(const char * str)
it != v.end();
++it)
{
- if (not FdOperation::is_diacritic(it->first) &&
- not FdOperation::is_diacritic(it->second))
+ if (! FdOperation::is_diacritic(it->first) &&
+ ! FdOperation::is_diacritic(it->second))
{ v_wo_flags.push_back(*it); }
}
return v_wo_flags;
diff --git a/libhfst/src/HfstSymbolDefs.h b/libhfst/src/HfstSymbolDefs.h
index 36461e2..26f5950 100644
--- a/libhfst/src/HfstSymbolDefs.h
+++ b/libhfst/src/HfstSymbolDefs.h
@@ -17,6 +17,8 @@
#include <map>
#include <set>
+#include "hfstdll.h"
+
/** @file HfstSymbolDefs.h
\brief Typedefs and functions for symbols, symbol pairs and
sets of symbols. */
@@ -104,14 +106,14 @@ namespace hfst
const std::string internal_default = "@_DEFAULT_SYMBOL_@";
/* Check whether a string is equal to reserved internal representation. */
- bool is_epsilon(std::string const & str);
- bool is_unknown(std::string const & str);
- bool is_identity(std::string const & str);
- bool is_default(std::string const & str);
- bool is_epsilon(const char * str);
- bool is_unknown(const char * str);
- bool is_identity(const char * str);
- bool is_default(const char * str);
+ HFSTDLL bool is_epsilon(std::string const & str);
+ HFSTDLL bool is_unknown(std::string const & str);
+ HFSTDLL bool is_identity(std::string const & str);
+ HFSTDLL bool is_default(std::string const & str);
+ HFSTDLL bool is_epsilon(const char * str);
+ HFSTDLL bool is_unknown(const char * str);
+ HFSTDLL bool is_identity(const char * str);
+ HFSTDLL bool is_default(const char * str);
/* For internal use */
typedef std::pair<unsigned int, unsigned int> NumberPair;
@@ -121,18 +123,19 @@ namespace hfst
typedef std::map<unsigned int,unsigned int> NumberNumberMap;
namespace symbols {
- void collect_unknown_sets(StringSet &s1, StringSet &unknown1,
+ HFSTDLL void collect_unknown_sets(StringSet &s1, StringSet &unknown1,
StringSet &s2, StringSet &unknown2);
- int longest_path_length(const hfst::HfstTwoLevelPaths & paths, bool equally_long=false);
- hfst::HfstTwoLevelPaths get_longest_paths(const hfst::HfstTwoLevelPaths & paths);
- StringVector to_string_vector(const hfst::HfstTwoLevelPath & path);
- std::string to_string(const StringVector & sv, bool spaces=false);
- std::string to_string(const StringPairVector & sv, bool spaces=false);
- hfst::HfstTwoLevelPaths remove_flags(const hfst::HfstTwoLevelPaths & paths);
- hfst::HfstTwoLevelPath remove_flags(const hfst::HfstTwoLevelPath & path);
- StringPairSet to_string_pair_set(const StringSet & ss);
- StringPairVector remove_flags(const StringPairVector &v);
- StringVector remove_flags(const StringVector &v);
+ HFSTDLL int longest_path_length(const hfst::HfstTwoLevelPaths & paths, bool equally_long=false);
+ HFSTDLL hfst::HfstTwoLevelPaths get_longest_paths(const hfst::HfstTwoLevelPaths & paths);
+ HFSTDLL StringVector to_string_vector(const hfst::HfstTwoLevelPath & path);
+ HFSTDLL std::string to_string(const StringVector & sv, bool spaces=false);
+ HFSTDLL std::string to_string(const StringPairVector & sv, bool spaces=false);
+ HFSTDLL StringVector to_string_vector(const StringPairVector & spv, bool input_side);
+ HFSTDLL hfst::HfstTwoLevelPaths remove_flags(const hfst::HfstTwoLevelPaths & paths);
+ HFSTDLL hfst::HfstTwoLevelPath remove_flags(const hfst::HfstTwoLevelPath & path);
+ HFSTDLL StringPairSet to_string_pair_set(const StringSet & ss);
+ HFSTDLL StringPairVector remove_flags(const StringPairVector &v);
+ HFSTDLL StringVector remove_flags(const StringVector &v);
}
}
#endif
diff --git a/libhfst/src/HfstTokenizer.cc b/libhfst/src/HfstTokenizer.cc
index 8dd27b3..a27c794 100644
--- a/libhfst/src/HfstTokenizer.cc
+++ b/libhfst/src/HfstTokenizer.cc
@@ -88,7 +88,7 @@ namespace hfst
int HfstTokenizer::get_next_symbol_size(const char * symbol)
const
{
- if (not *symbol)
+ if (! *symbol)
{ return 0; }
const char * multi_char_symbol_end = multi_char_symbols.find(symbol);
@@ -109,7 +109,7 @@ const
}
bool HfstTokenizer::is_skip_symbol(hfst::String &s) const
-{ return (s == "") or (skip_symbol_set.find(s) != skip_symbol_set.end()); }
+{ return (s == "") || (skip_symbol_set.find(s) != skip_symbol_set.end()); }
void
HfstTokenizer::add_multichar_symbol(const string& symbol)
@@ -374,8 +374,8 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
size_t additional_chars = 0;
// The bytes 192, 193, 245, 246 and 247 are invalid in utf8.
- if (initial_char == 192 or initial_char == 193 or
- initial_char == 245 or initial_char == 246 or initial_char == 247)
+ if (initial_char == 192 || initial_char == 193 ||
+ initial_char == 245 || initial_char == 246 || initial_char == 247)
{ HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
"leading octet in [192, 193, 245, 246, 247]"); }
// Case 0xxxxxxx, i.e. ASCII byte.
@@ -409,7 +409,7 @@ StringPairVector HfstTokenizer::tokenize_and_align_flag_diacritics
"eos in multioctet sequence"); }
unsigned char byte = *it;
// All continuation bytes look like 10xxxxxx.
- if (not (128 & byte and 64 ^ byte))
+ if (! (128 & byte && 64 ^ byte))
{ HFST_THROW_MESSAGE(IncorrectUtf8CodingException,
"not continuation octet & 100000000b"); }
}
diff --git a/libhfst/src/HfstTokenizer.h b/libhfst/src/HfstTokenizer.h
index d8e0cf5..a86d5ab 100644
--- a/libhfst/src/HfstTokenizer.h
+++ b/libhfst/src/HfstTokenizer.h
@@ -18,6 +18,8 @@
#include <climits>
#include <string>
+#include "hfstdll.h"
+
/** @file HfstTokenizer.h
\brief Declaration of class #hfst::HfstTokenizer. */
@@ -50,10 +52,10 @@ namespace hfst
MultiCharSymbolTrie * get_symbol_rest_trie(const char * p) const;
public:
- MultiCharSymbolTrie(void);
- ~MultiCharSymbolTrie(void);
- void add(const char * p);
- const char * find(const char * p) const;
+ HFSTDLL MultiCharSymbolTrie(void);
+ HFSTDLL ~MultiCharSymbolTrie(void);
+ HFSTDLL void add(const char * p);
+ HFSTDLL const char * find(const char * p) const;
};
/** \brief A tokenizer for creating transducers from UTF-8 strings.
@@ -93,7 +95,7 @@ namespace hfst
public:
/** \brief Create a tokenizer that recognizes utf-8 symbols. */
- HfstTokenizer();
+ HFSTDLL HfstTokenizer();
/** \brief Add a symbol to be skipped to this tokenizer.
@@ -101,7 +103,7 @@ namespace hfst
For example if we have a multicharacter symbol "foo" and a
skip symbol "bar", the string "fobaro" will be tokenized
"f" "o" "o", not "foo". */
- void add_skip_symbol(const std::string &symbol);
+ HFSTDLL void add_skip_symbol(const std::string &symbol);
/** \brief Add a multicharacter symbol \a symbol to this tokenizer.
@@ -109,15 +111,15 @@ namespace hfst
not considered a multicharacter symbol. For example if we have
a multicharacter symbol "foo" and a skip symbol "bar", the string
"fobaro" will be tokenized "f" "o" "o", not "foo". */
- void add_multichar_symbol(const std::string& symbol);
+ HFSTDLL void add_multichar_symbol(const std::string& symbol);
/** \brief Tokenize the string \a input_string. */
- StringPairVector tokenize(const std::string &input_string) const;
+ HFSTDLL StringPairVector tokenize(const std::string &input_string) const;
/** \brief Tokenize the string \a input_string. */
- StringVector tokenize_one_level(const std::string &input_string) const;
+ HFSTDLL StringVector tokenize_one_level(const std::string &input_string) const;
- static StringPairVector tokenize_space_separated(const std::string & str);
+ HFSTDLL static StringPairVector tokenize_space_separated(const std::string & str);
/** \brief Tokenize the string pair \a input_string : \a output_string.
@@ -125,14 +127,14 @@ namespace hfst
inserted to the end of the tokenized string with less tokens
so that both tokenized strings have the same number of tokens.
*/
- StringPairVector tokenize(const std::string &input_string,
+ HFSTDLL StringPairVector tokenize(const std::string &input_string,
const std::string &output_string) const;
- StringPairVector tokenize(const std::string &input_string,
+ HFSTDLL StringPairVector tokenize(const std::string &input_string,
const std::string &output_string,
void (*warn_about_pair)(const std::pair<std::string, std::string> &symbol_pair)) const;
- StringPairVector tokenize_and_align_flag_diacritics
+ HFSTDLL StringPairVector tokenize_and_align_flag_diacritics
(const std::string &input_string,
const std::string &output_string,
void (*warn_about_pair)(const std::pair<std::string, std::string> &symbol_pair)) const;
@@ -154,7 +156,7 @@ namespace hfst
//! -# Initial bytes 11110xxx are followed by exactly three
//! continuation bytes.
//! (For reference: http://en.wikipedia.org/wiki/UTF-8)
- static void check_utf8_correctness(const std::string &input_string);
+ HFSTDLL static void check_utf8_correctness(const std::string &input_string);
};
}
#endif
diff --git a/libhfst/src/HfstTransducer.cc b/libhfst/src/HfstTransducer.cc
index d7c0083..3a3429c 100644
--- a/libhfst/src/HfstTransducer.cc
+++ b/libhfst/src/HfstTransducer.cc
@@ -95,8 +95,25 @@ hfst::implementations::FomaTransducer HfstTransducer::foma_interface;
//
// -----------------------------------------------------------------------
+#if HAVE_XFSM
+ void initialize_xfsm()
+ {
+ XfsmTransducer::initialize_xfsm();
+ }
+
+ InitializeXfsm::InitializeXfsm()
+ {
+ initialize_xfsm();
+ }
+
+ InitializeXfsm dummy;
+#endif
+
/* The default minimization algorithm if Hopcroft. */
MinimizationAlgorithm minimization_algorithm=HOPCROFT;
+ /* By default, we do not minimize transducers that are already minimal.
+ This variable is for debugging and profiling. */
+bool minimize_even_if_already_minimal=false;
/* By default, weights are not encoded in minimization. */
bool encode_weights=false;
/* By default, harmonization is not optimized. */
@@ -117,8 +134,22 @@ bool flag_is_epsilon_in_composition=false;
return xerox_composition;
}
+ void set_minimize_even_if_already_minimal(bool value) {
+ minimize_even_if_already_minimal=value;
+#if HAVE_XFSM
+ XfsmTransducer::set_minimize_even_if_already_minimal(value);
+#endif
+ }
+
+ bool get_minimize_even_if_already_minimal() {
+ return minimize_even_if_already_minimal;
+ }
+
void set_flag_is_epsilon_in_composition(bool value) {
flag_is_epsilon_in_composition=value;
+#if HAVE_XFSM
+ XfsmTransducer::set_compose_flag_as_special(value);
+#endif
}
bool get_flag_is_epsilon_in_composition() {
@@ -209,10 +240,21 @@ void HfstTransducer::insert_to_alphabet(const std::string &symbol)
if (symbol == "")
HFST_THROW_MESSAGE(EmptyStringException, "insert_to_alphabet");
- hfst::implementations::HfstBasicTransducer * net
- = convert_to_basic_transducer();
- net->add_symbol_to_alphabet(symbol);
- convert_to_hfst_transducer(net);
+ if (this->type != XFSM_TYPE)
+ {
+ hfst::implementations::HfstBasicTransducer * net
+ = convert_to_basic_transducer();
+ net->add_symbol_to_alphabet(symbol);
+ convert_to_hfst_transducer(net);
+ }
+ else
+ {
+#if HAVE_XFSM
+ this->xfsm_interface.add_symbol_to_alphabet(this->implementation.xfsm, symbol);
+#else
+ HFST_THROW(ImplementationTypeNotAvailableException);
+#endif
+ }
}
void HfstTransducer::insert_to_alphabet(const std::set<std::string> &symbols)
@@ -225,10 +267,21 @@ void HfstTransducer::insert_to_alphabet(const std::set<std::string> &symbols)
{ HFST_THROW_MESSAGE(EmptyStringException, "insert_to_alphabet"); }
}
- hfst::implementations::HfstBasicTransducer * net
- = convert_to_basic_transducer();
- net->add_symbols_to_alphabet(symbols);
- convert_to_hfst_transducer(net);
+ if (this->type != XFSM_TYPE)
+ {
+ hfst::implementations::HfstBasicTransducer * net
+ = convert_to_basic_transducer();
+ net->add_symbols_to_alphabet(symbols);
+ convert_to_hfst_transducer(net);
+ }
+ else
+ {
+#if HAVE_XFSM
+ this->xfsm_interface.add_symbols_to_alphabet(this->implementation.xfsm, symbols);
+#else
+ HFST_THROW(ImplementationTypeNotAvailableException);
+#endif
+ }
}
@@ -256,6 +309,16 @@ void HfstTransducer::remove_from_alphabet(const std::set<std::string> &symbols)
this->remove_from_alphabet(*it);
}
}
+
+/* Implemented for XFSM_TYPE, as conversion between HfstBasicFormat and XFSM_TYPE is slow. */
+void HfstTransducer::remove_symbols_from_alphabet(const StringSet & symbols)
+{
+ if (this->type != XFSM_TYPE)
+ HFST_THROW_MESSAGE(FunctionNotImplementedException, "remove_symbols_from_alphabet");
+#if HAVE_XFSM
+ this->xfsm_interface.remove_symbols_from_alphabet(this->implementation.xfsm, symbols);
+#endif
+}
HfstTransducer &HfstTransducer::prune_alphabet(bool force)
@@ -285,6 +348,10 @@ StringSet HfstTransducer::get_first_input_symbols() const
case FOMA_TYPE:
HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_first_input_symbols");
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_first_input_symbols");
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
case HFST_OL_TYPE:
@@ -316,6 +383,10 @@ StringSet HfstTransducer::get_alphabet() const
case FOMA_TYPE:
return foma_interface.get_alphabet(implementation.foma);
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ return xfsm_interface.get_alphabet(implementation.xfsm);
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
case HFST_OL_TYPE:
@@ -349,6 +420,10 @@ unsigned int HfstTransducer::get_symbol_number(const std::string &symbol)
return foma_interface.get_symbol_number(implementation.foma,
symbol);
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ HFST_THROW_MESSAGE(FunctionNotImplementedException, "get_symbol_number");
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
case HFST_OL_TYPE:
@@ -442,6 +517,12 @@ HfstTransducer * HfstTransducer::harmonize_(const HfstTransducer &another)
return NULL;
break;
#endif // HAVE_FOMA
+#if HAVE_XFSM
+ case (XFSM_TYPE):
+ // no need to harmonize as xfsm's functions take care of harmonizing
+ return NULL;
+ break;
+#endif // HAVE_XFSM
#if HAVE_SFST || HAVE_OPENFST
case (SFST_TYPE):
case (TROPICAL_OPENFST_TYPE):
@@ -517,6 +598,11 @@ void HfstTransducer::harmonize(HfstTransducer &another)
// no need to harmonize as foma's functions take care of harmonizing
break;
#endif // HAVE_FOMA
+#if HAVE_XFSM
+ case (XFSM_TYPE):
+ // no need to harmonize as xfsm's functions take care of harmonizing
+ break;
+#endif // HAVE_XFSM
#if HAVE_SFST || HAVE_OPENFST
case (SFST_TYPE):
case (TROPICAL_OPENFST_TYPE):
@@ -566,6 +652,12 @@ void HfstTransducer::print_alphabet()
net.print_alphabet();
}
#endif
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ {
+ HFST_THROW_MESSAGE(FunctionNotImplementedException, "print_alphabet");
+ }
+#endif
return;
}
@@ -742,7 +834,7 @@ HfstTransducer::HfstTransducer():
HfstTransducer::HfstTransducer(ImplementationType type):
type(type),anonymous(false),is_trie(true), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
switch (type)
@@ -770,6 +862,11 @@ HfstTransducer::HfstTransducer(ImplementationType type):
implementation.foma = foma_interface.create_empty_transducer();
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm = xfsm_interface.create_empty_transducer();
+ break;
+#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
@@ -796,7 +893,7 @@ HfstTransducer::HfstTransducer(const std::string& utf8_str,
ImplementationType type):
type(type),anonymous(false),is_trie(true), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
if (utf8_str == "")
@@ -833,6 +930,12 @@ HfstTransducer::HfstTransducer(const std::string& utf8_str,
foma_interface.define_transducer(spv);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ xfsm_interface.define_transducer(spv);
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(SpecifiedTypeRequiredException);
default:
@@ -844,7 +947,7 @@ HfstTransducer::HfstTransducer(const StringPairVector & spv,
ImplementationType type):
type(type), anonymous(false), is_trie(false), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
for (StringPairVector::const_iterator it = spv.begin();
@@ -885,6 +988,13 @@ HfstTransducer::HfstTransducer(const StringPairVector & spv,
this->type = FOMA_TYPE;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ xfsm_interface.define_transducer(spv);
+ this->type = XFSM_TYPE;
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(SpecifiedTypeRequiredException);
default:
@@ -897,7 +1007,7 @@ HfstTransducer::HfstTransducer(const StringPairSet & sps,
bool cyclic):
type(type),anonymous(false),is_trie(false), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
for (StringPairSet::const_iterator it = sps.begin();
@@ -938,6 +1048,13 @@ HfstTransducer::HfstTransducer(const StringPairSet & sps,
this->type = FOMA_TYPE;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ xfsm_interface.define_transducer(sps,cyclic);
+ this->type = XFSM_TYPE;
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(SpecifiedTypeRequiredException);
default:
@@ -949,7 +1066,7 @@ HfstTransducer::HfstTransducer(const std::vector<StringPairSet> & spsv,
ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
for (std::vector<StringPairSet>::const_iterator it = spsv.begin();
@@ -995,6 +1112,13 @@ HfstTransducer::HfstTransducer(const std::vector<StringPairSet> & spsv,
this->type = FOMA_TYPE;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ xfsm_interface.define_transducer(spsv);
+ this->type = XFSM_TYPE;
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(SpecifiedTypeRequiredException);
default:
@@ -1009,7 +1133,7 @@ HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
ImplementationType type):
type(type),anonymous(false),is_trie(true), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
if (upper_utf8_str == "" ||
@@ -1048,6 +1172,12 @@ HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
foma_interface.define_transducer(spv);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ xfsm_interface.define_transducer(spv);
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(SpecifiedTypeRequiredException);
default:
@@ -1059,7 +1189,7 @@ HfstTransducer::HfstTransducer(const std::string& upper_utf8_str,
HfstTransducer::HfstTransducer(HfstInputStream &in):
type(in.type), anonymous(false),is_trie(false), name("")
{
- if (not is_implementation_type_available(type)) {
+ if (! is_implementation_type_available(type)) {
HFST_THROW(ImplementationTypeNotAvailableException);
}
@@ -1070,7 +1200,7 @@ HfstTransducer::HfstTransducer(const HfstTransducer &another):
type(another.type),anonymous(another.anonymous),
is_trie(another.is_trie), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
for (map<string,string>::const_iterator prop = another.props.begin();
prop != another.props.end();
@@ -1105,6 +1235,11 @@ HfstTransducer::HfstTransducer(const HfstTransducer &another):
implementation.foma = foma_interface.copy(another.implementation.foma);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm = xfsm_interface.copy(another.implementation.xfsm);
+ break;
+#endif
case HFST_OL_TYPE:
implementation.hfst_ol
= another.implementation.hfst_ol->copy
@@ -1127,7 +1262,7 @@ HfstTransducer::HfstTransducer
ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
switch (type)
@@ -1156,6 +1291,13 @@ HfstTransducer::HfstTransducer
ConversionFunctions::hfst_basic_transducer_to_foma(&net);
break;
#endif
+#if HAVE_XFSM
+ // this is slow!
+ case XFSM_TYPE:
+ implementation.xfsm =
+ ConversionFunctions::hfst_basic_transducer_to_xfsm(&net);
+ break;
+#endif
case HFST_OL_TYPE:
implementation.hfst_ol =
ConversionFunctions::hfst_basic_transducer_to_hfst_ol(&net, false);
@@ -1173,7 +1315,7 @@ HfstTransducer::HfstTransducer
HfstTransducer::~HfstTransducer(void)
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
switch (type)
@@ -1198,6 +1340,12 @@ HfstTransducer::~HfstTransducer(void)
foma_interface.delete_foma(implementation.foma);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ // FIX THIS: causes errors...
+ //delete implementation.xfsm;
+ break;
+#endif
case HFST_OL_TYPE:
case HFST_OLW_TYPE:
delete implementation.hfst_ol;
@@ -1214,7 +1362,7 @@ HfstTransducer::HfstTransducer(const std::string &symbol,
ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
HfstTokenizer::check_utf8_correctness(symbol);
@@ -1249,6 +1397,12 @@ HfstTransducer::HfstTransducer(const std::string &symbol,
// should the char* be deleted?
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm = xfsm_interface.define_transducer(symbol);
+ // should the char* be deleted?
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
default:
@@ -1261,7 +1415,7 @@ HfstTransducer::HfstTransducer(const std::string &isymbol,
ImplementationType type):
type(type),anonymous(false),is_trie(false), name("")
{
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
HfstTokenizer::check_utf8_correctness(isymbol);
@@ -1301,6 +1455,13 @@ HfstTransducer::HfstTransducer(const std::string &isymbol,
// should the char*:s be deleted?
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm
+ = xfsm_interface.define_transducer(isymbol, osymbol);
+ // should the char*:s be deleted?
+ break;
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
default:
@@ -1376,7 +1537,7 @@ bool HfstTransducer::compare(const HfstTransducer &another, bool harmonize) cons
one_copy.insert_missing_symbols_to_alphabet_from(another_copy, true);
another_copy.insert_missing_symbols_to_alphabet_from(one_copy, true);
- if (this->type != FOMA_TYPE)
+ if (this->type != FOMA_TYPE && this->type != XFSM_TYPE)
{
HfstTransducer *tmp = one_copy.harmonize_(another_copy);
another_copy = *tmp;
@@ -1411,6 +1572,12 @@ bool HfstTransducer::compare(const HfstTransducer &another, bool harmonize) cons
one_copy.implementation.foma,
another_copy.implementation.foma);
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ return one_copy.xfsm_interface.are_equivalent(
+ one_copy.implementation.xfsm,
+ another_copy.implementation.xfsm);
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
default:
@@ -1443,6 +1610,10 @@ bool HfstTransducer::is_automaton(void) const
return t.is_automaton();
}
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ HFST_THROW(FunctionNotImplementedException);
+#endif
case ERROR_TYPE:
HFST_THROW(TransducerHasWrongTypeException);
default:
@@ -1470,6 +1641,10 @@ bool HfstTransducer::is_cyclic(void) const
case FOMA_TYPE:
return foma_interface.is_cyclic(implementation.foma);
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ return xfsm_interface.is_cyclic(implementation.xfsm);
+#endif
case HFST_OL_TYPE:
case HFST_OLW_TYPE:
return hfst_ol_interface.is_cyclic(implementation.hfst_ol);
@@ -1497,6 +1672,11 @@ unsigned int HfstTransducer::number_of_states() const
return this->foma_interface.number_of_states
(this->implementation.foma);
#endif
+#if HAVE_XFSM
+ if (type == XFSM_TYPE)
+ return this->xfsm_interface.number_of_states
+ (this->implementation.xfsm);
+#endif
return 0;
}
@@ -1517,6 +1697,11 @@ unsigned int HfstTransducer::number_of_arcs() const
return this->foma_interface.number_of_arcs
(this->implementation.foma);
#endif
+#if HAVE_XFSM
+ if (type == XFSM_TYPE)
+ return this->xfsm_interface.number_of_arcs
+ (this->implementation.xfsm);
+#endif
return 0;
}
@@ -1784,6 +1969,7 @@ static HfstTransducer * get_flag_filter
static void flag_purge(HfstTransducer & transducer, const std::string & flag)
{
ImplementationType type = transducer.get_type();
+ // slow for xfsm_transducer..
HfstBasicTransducer net(transducer);
net.flag_purge(flag);
transducer = HfstTransducer(net, type);
@@ -1800,6 +1986,13 @@ HfstTransducer &HfstTransducer::eliminate_flags()
return *this;
}
#endif
+#if HAVE_XFSM
+ if (type == XFSM_TYPE)
+ {
+ this->xfsm_interface.eliminate_flags_xfsm(this->implementation.xfsm);
+ return *this;
+ }
+#endif
HfstBasicTransducer basic(*this);
StringSet flags = basic.get_flags();
@@ -1831,6 +2024,13 @@ HfstTransducer &HfstTransducer::eliminate_flag(const std::string & flag)
return *this;
}
#endif
+#if HAVE_XFSM
+ if (type == XFSM_TYPE)
+ {
+ this->xfsm_interface.eliminate_flag_xfsm(this->implementation.xfsm, flag);
+ return *this;
+ }
+#endif
HfstBasicTransducer basic(*this);
StringSet flags = basic.get_flags();
@@ -1866,6 +2066,9 @@ HfstTransducer &HfstTransducer::remove_epsilons()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::remove_epsilons,
#endif
+#if HAVE_XFSM
+ NULL,
+#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//&hfst::implementations::MyTransducerLibraryTransducer::remove_epsilons,
@@ -1875,6 +2078,7 @@ HfstTransducer &HfstTransducer::remove_epsilons()
HfstTransducer &HfstTransducer::prune()
{
#if HAVE_OPENFST
+ // slow for xfsm type...
this->convert(TROPICAL_OPENFST_TYPE);
fst::StdVectorFst * temp = hfst::implementations::TropicalWeightTransducer::prune
(this->implementation.tropical_ofst);
@@ -1887,6 +2091,10 @@ HfstTransducer &HfstTransducer::prune()
HfstTransducer &HfstTransducer::determinize()
{ is_trie = false;
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE) {
+ HFST_THROW(FunctionNotImplementedException); }
+#endif
return apply(
#if HAVE_SFST
&hfst::implementations::SfstTransducer::determinize,
@@ -1900,6 +2108,9 @@ HfstTransducer &HfstTransducer::determinize()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::determinize,
#endif
+#if HAVE_XFSM
+ NULL,
+#endif
/* Add here your implementation. */
false ); }
@@ -1918,6 +2129,9 @@ HfstTransducer &HfstTransducer::minimize()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::minimize,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::minimize,
+#endif
/* Add here your implementation. */
false );
}
@@ -1945,6 +2159,9 @@ HfstTransducer &HfstTransducer::repeat_star()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::repeat_star,
#endif
+#if HAVE_XFSM
+ NULL,
+#endif
/* Add here your implementation. */
false ); }
@@ -1963,6 +2180,9 @@ HfstTransducer &HfstTransducer::repeat_plus()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::repeat_plus,
#endif
+#if HAVE_XFSM
+ NULL,
+#endif
/* Add here your implementation. */
false ); }
@@ -1981,6 +2201,9 @@ HfstTransducer &HfstTransducer::repeat_n(unsigned int n)
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::repeat_n,
#endif
+#if HAVE_XFSM
+ NULL,
+#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//&hfst::implementations::MyTransducerLibraryTransducer::repeat_n,
@@ -1989,6 +2212,13 @@ HfstTransducer &HfstTransducer::repeat_n(unsigned int n)
HfstTransducer &HfstTransducer::repeat_n_plus(unsigned int n)
{ is_trie = false; // This could be done so that is_trie is preserved
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ {
+ this->xfsm_interface.repeat_n_plus(this->implementation.xfsm, n);
+ return *this;
+ }
+#endif
HfstTransducer a(*this);
return (this->repeat_n(n).concatenate(a.repeat_star()));
}
@@ -2008,11 +2238,21 @@ HfstTransducer &HfstTransducer::repeat_n_minus(unsigned int n)
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::repeat_le_n,
#endif
+#if HAVE_XFSM
+ NULL,
+#endif
/* Add here your implementation. */
n ); }
HfstTransducer &HfstTransducer::repeat_n_to_k(unsigned int n, unsigned int k)
{ is_trie = false; // This could be done so that is_trie is preserved
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ {
+ this->xfsm_interface.repeat_n_to_k(this->implementation.xfsm, n, k);
+ return *this;
+ }
+#endif
HfstTransducer a(*this);
return (this->repeat_n(n).concatenate(a.repeat_n_minus(k-n)));
}
@@ -2040,6 +2280,9 @@ HfstTransducer &HfstTransducer::optionalize()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::optionalize,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::optionalize,
+#endif
/* Add here your implementation. */
false ); }
@@ -2058,6 +2301,9 @@ HfstTransducer &HfstTransducer::invert()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::invert,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::invert,
+#endif
/* Add here your implementation. */
false ); }
@@ -2076,6 +2322,9 @@ HfstTransducer &HfstTransducer::reverse()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::reverse,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::reverse,
+#endif
/* Add here your implementation. */
false ); }
@@ -2094,6 +2343,9 @@ HfstTransducer &HfstTransducer::input_project()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::extract_input_language,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::extract_input_language,
+#endif
/* Add here your implementation. */
false ); }
@@ -2113,6 +2365,9 @@ HfstTransducer &HfstTransducer::output_project()
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::extract_output_language,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::extract_output_language,
+#endif
/* Add here your implementation. */
false ); }
@@ -2512,7 +2767,7 @@ void HfstTransducer::extract_random_paths_fd
HfstTransducer &HfstTransducer::n_best(unsigned int n)
{
- if (not is_implementation_type_available(TROPICAL_OPENFST_TYPE)) {
+ if (! is_implementation_type_available(TROPICAL_OPENFST_TYPE)) {
(void)n;
HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
"HfstTransducer::n_best implemented only for "
@@ -2575,6 +2830,25 @@ bool HfstTransducer::is_special_symbol(const std::string &symbol)
return false;
}
+StringSet HfstTransducer::insert_missing_diacritics_to_alphabet_from(const HfstTransducer &another)
+{
+ StringSet this_alphabet = this->get_alphabet();
+ StringSet another_alphabet = another.get_alphabet();
+ StringSet missing_flags;
+
+ for (StringSet::const_iterator it = another_alphabet.begin();
+ it != another_alphabet.end(); it++)
+ {
+ if (this_alphabet.find(*it) == this_alphabet.end())
+ {
+ if (FdOperation::is_diacritic(*it))
+ missing_flags.insert(*it);
+ }
+ }
+ this->insert_to_alphabet(missing_flags);
+ return missing_flags;
+}
+
void HfstTransducer::insert_missing_symbols_to_alphabet_from(const HfstTransducer &another, bool only_special_symbols)
{
StringSet this_alphabet = this->get_alphabet();
@@ -2803,13 +3077,16 @@ void HfstTransducer::twosided_flag_diacritics()
std::string encode_flag(const std::string &flag_diacritic)
{
std::string retval(flag_diacritic);
- retval[0] = '$';
- retval[retval.size()-1] = '$';
+ retval[0] = '%';
+ retval[retval.size()-1] = '%';
return retval;
}
std::string decode_flag(const std::string &flag_diacritic)
{
+ if (flag_diacritic[0] != '%' || flag_diacritic[flag_diacritic.size()-1] != '%')
+ return std::string(flag_diacritic);
+
std::string retval(flag_diacritic);
retval[0] = '@';
retval[retval.size()-1] = '@';
@@ -2860,6 +3137,20 @@ void encode_flag_diacritics(HfstTransducer &fst)
StringSet alpha = basic_fst.get_alphabet();
for (StringSet::const_iterator it = alpha.begin(); it != alpha.end(); it++)
{
+ if (it->size() > 4)
+ {
+ if ((it->at(0) == '%') && (it->at(it->size()-1) == '%'))
+ {
+ std::string str(*it);
+ str[0] = '@';
+ str[str.size()-1] = '@';
+ if (FdOperation::is_diacritic(str))
+ {
+ std::string msg = "error: reserved symbol '" + str + "' detected";
+ throw msg.c_str();
+ }
+ }
+ }
String symbol = *it;
if (FdOperation::is_diacritic(symbol))
symbol = encode_flag(symbol);
@@ -2951,7 +3242,7 @@ void HfstTransducer::harmonize_flag_diacritics(HfstTransducer &another,
bool this_has_flag_diacritics = has_flags(*this);
bool another_has_flag_diacritics = has_flags(another);
- if (this_has_flag_diacritics and another_has_flag_diacritics)
+ if (this_has_flag_diacritics && another_has_flag_diacritics)
{
rename_flag_diacritics(*this,"_1");
rename_flag_diacritics(another,"_2");
@@ -2963,9 +3254,9 @@ void HfstTransducer::harmonize_flag_diacritics(HfstTransducer &another,
this->remove_illegal_flag_paths();
}
}
- else if (this_has_flag_diacritics and insert_renamed_flags)
+ else if (this_has_flag_diacritics && insert_renamed_flags)
{ another.insert_freely_missing_flags_from(*this); }
- else if (another_has_flag_diacritics and insert_renamed_flags)
+ else if (another_has_flag_diacritics && insert_renamed_flags)
{ this->insert_freely_missing_flags_from(another); }
}
@@ -3055,6 +3346,15 @@ HfstTransducer &HfstTransducer::insert_freely
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"HfstTransducer::insert_freely");
+ // Segfaults in xfst command line tool...
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ {
+ this->xfsm_interface.insert_freely(this->implementation.xfsm, tr.implementation.xfsm);
+ return *this;
+ }
+#endif
+
/* In this function, this transducer must always be harmonized
according to tr, not the other way round. */
HfstTransducer * tr_harmonized = NULL;
@@ -3188,6 +3488,10 @@ HfstTransducer &HfstTransducer::insert_freely
HfstTransducer &HfstTransducer::substitute
(bool (*func)(const StringPair &sp, StringPairSet &sps))
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->substitute(func);
@@ -3198,6 +3502,10 @@ HfstTransducer &HfstTransducer::substitute
(const std::string &old_symbol, const std::string &new_symbol,
bool input_side, bool output_side)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
// empty strings are not accepted
if (old_symbol == "" || new_symbol == "")
HFST_THROW_MESSAGE
@@ -3256,6 +3564,10 @@ HfstTransducer &HfstTransducer::substitute
(const StringPair &old_symbol_pair,
const StringPair &new_symbol_pair)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
// empty strings are not accepted
if (old_symbol_pair.first == "" || old_symbol_pair.second == "" ||
new_symbol_pair.first == "" || new_symbol_pair.second == "")
@@ -3274,6 +3586,10 @@ HfstTransducer &HfstTransducer::substitute
(const StringPair &old_symbol_pair,
const StringPairSet &new_symbol_pair_set)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
if(old_symbol_pair.first == "" || old_symbol_pair.second == "")
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -3293,6 +3609,10 @@ HfstTransducer &HfstTransducer::substitute_symbols
HfstTransducer &HfstTransducer::substitute
(const HfstSymbolSubstitutions &substitutions)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
@@ -3318,6 +3638,10 @@ HfstTransducer &HfstTransducer::substitute_symbol_pairs
HfstTransducer &HfstTransducer::substitute
(const HfstSymbolPairSubstitutions &substitutions)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
hfst::implementations::HfstBasicTransducer * net
= convert_to_basic_transducer();
net->substitute(substitutions);
@@ -3328,6 +3652,10 @@ HfstTransducer &HfstTransducer::substitute
(const StringPair &symbol_pair,
HfstTransducer &transducer, bool harmonize)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
if (this->type != transducer.type) {
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"HfstTransducer::substitute"); }
@@ -3559,6 +3887,10 @@ bool substitute_unknown_identity_pairs
HfstTransducer &HfstTransducer::merge
(const HfstTransducer &another, const struct hfst::xre::XreConstructorArguments & args)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
HfstBasicTransducer this_basic(*this);
HfstBasicTransducer another_basic(another);
std::set<std::string> markers_added;
@@ -3610,7 +3942,16 @@ HfstTransducer &HfstTransducer::compose
another_copy->convert(this->type);
}
- if (flag_is_epsilon_in_composition)
+ /* If we want flag diacritcs to be handled in the same way as epsilons
+ in composition, we substitute output flags of first transducer with
+ epsilons and input flags of second transducer with epsilons. NOTE:
+ we assume that flag diacritics are always used as identities, i.e.
+ @FLAG1@:foo, foo:@FLAG2@ or @FLAG1@:@FLAG2@ are not allowed in
+ transitions.
+
+ In XFSM, this is controlled through a variable that has been set
+ already when set_flag_is_epsilon_in_composition() has been called */
+ if (flag_is_epsilon_in_composition && this->type != XFSM_TYPE)
{
try
{
@@ -3623,24 +3964,55 @@ HfstTransducer &HfstTransducer::compose
}
}
+ // Variables possibly needed next.
+ StringSet diacritics_added_from_another_to_this;
+ StringSet diacritics_added_from_this_to_another;
+
+ /* If we want flag diacritics to match with identities and unknowns (as
+ Xerox does), we encode flags as normal symbols before composition. We do
+ not need to do this for XFSM transducers, as this is already the default
+ behavior.
+
+ If we do not want flags to match with identities and unknowns (the default
+ for non-XFSM transducers) and the transducer is of XFSM_TYPE, we add flags
+ of first transducer to alphabet of second transducer if they are not yet
+ there and vice versa before composition. */
if (xerox_composition)
{
- encode_flag_diacritics(*this);
- encode_flag_diacritics(*another_copy);
+ if (this->type != XFSM_TYPE)
+ {
+ encode_flag_diacritics(*this);
+ encode_flag_diacritics(*another_copy);
+ }
+ }
+ else
+ {
+ if (this->type == XFSM_TYPE)
+ {
+ diacritics_added_from_another_to_this
+ = this->insert_missing_diacritics_to_alphabet_from(*another_copy);
+ diacritics_added_from_this_to_another
+ = another_copy->insert_missing_diacritics_to_alphabet_from(*this);
+ }
}
- /* prevent harmonization, if needed */
+
+ /* Prevent harmonization (i.e. matching unknown symbols), if requested. */
if (! harmonize)
{
this->insert_missing_symbols_to_alphabet_from(*another_copy);
another_copy->insert_missing_symbols_to_alphabet_from(*this);
}
- /* special symbols are never harmonized */
+ // hfst-lexc's @_REG.Root_#_@ ??
+ /* Special symbols are never harmonized, add them to alphabets of
+ both transducers to prevent them from being matched with unknowns
+ and identities. */
this->insert_missing_symbols_to_alphabet_from(*another_copy, true);
another_copy->insert_missing_symbols_to_alphabet_from(*this, true);
- if (this->type != FOMA_TYPE)
+ // Harmonize, FOMA and XFSM take care of this by default.
+ if (this->type != FOMA_TYPE && this->type != XFSM_TYPE)
{
HfstTransducer * tmp =
this->harmonize_(const_cast<HfstTransducer&>(*another_copy));
@@ -3648,15 +4020,15 @@ HfstTransducer &HfstTransducer::compose
another_copy = tmp;
}
- // Handle special symbols here.
- if ( (this->type != FOMA_TYPE) && unknown_symbols_in_use)
- {
- // comment...
- this->substitute("@_IDENTITY_SYMBOL_@","@_UNKNOWN_SYMBOL_@",false,true);
- const_cast<HfstTransducer*>(another_copy)->substitute
- ("@_IDENTITY_SYMBOL_@","@_UNKNOWN_SYMBOL_@",true,false);
- }
-
+ /* Take care of unknown and identity symbols being handled right in
+ composition, FOMA and XFSM take care of this by default. */
+ if ( (this->type != FOMA_TYPE && this->type != XFSM_TYPE) && unknown_symbols_in_use)
+ {
+ this->substitute("@_IDENTITY_SYMBOL_@","@_UNKNOWN_SYMBOL_@",false,true);
+ const_cast<HfstTransducer*>(another_copy)->substitute
+ ("@_IDENTITY_SYMBOL_@","@_UNKNOWN_SYMBOL_@",true,false);
+ }
+
switch (this->type)
{
#if HAVE_SFST
@@ -3705,6 +4077,15 @@ HfstTransducer &HfstTransducer::compose
break;
}
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ {
+ this->implementation.xfsm =
+ this->xfsm_interface.compose(this->implementation.xfsm,
+ another_copy->implementation.xfsm);
+ break;
+ }
+#endif
#if HAVE_HFSTOL
case HFST_OL_TYPE:
case HFST_OLW_TYPE:
@@ -3717,20 +4098,32 @@ HfstTransducer &HfstTransducer::compose
HFST_THROW(FunctionNotImplementedException);
}
+ // Revert changes made before composition
+
if (xerox_composition)
{
- decode_flag_diacritics(*this);
- decode_flag_diacritics(*another_copy);
+ if (this->type != XFSM_TYPE)
+ {
+ decode_flag_diacritics(*this);
+ decode_flag_diacritics(*another_copy);
+ }
+ }
+ else
+ {
+ if (this->type == XFSM_TYPE)
+ {
+ this->remove_symbols_from_alphabet(diacritics_added_from_another_to_this);
+ another_copy->remove_symbols_from_alphabet(diacritics_added_from_this_to_another);
+ }
}
- if (flag_is_epsilon_in_composition)
+ if (flag_is_epsilon_in_composition && this->type != XFSM_TYPE)
{
this->substitute(&substitute_one_sided_flags);
}
- if ( (this->type != FOMA_TYPE) && unknown_symbols_in_use)
+ if ( (this->type != FOMA_TYPE && this->type != XFSM_TYPE) && unknown_symbols_in_use)
{
- // comment...
this->substitute(&substitute_single_identity_with_the_other_symbol);
(const_cast<HfstTransducer*>(another_copy))->
substitute(&substitute_unknown_identity_pairs);
@@ -3831,7 +4224,7 @@ HfstTransducer &HfstTransducer::remove_illegal_flag_paths(void)
it != alphabet.end();
++it)
{
- if (not FdOperation::is_diacritic(*it))
+ if (! FdOperation::is_diacritic(*it))
{ continue; }
if (it->find("_1.") != std::string::npos)
@@ -3847,7 +4240,7 @@ HfstTransducer &HfstTransducer::remove_illegal_flag_paths(void)
// if there aren't both _1 and _2 flag diaciritcs, there can be no
// illegal paths.
- if (_1_flags.empty() or _2_flags.empty())
+ if (_1_flags.empty() || _2_flags.empty())
{ return *this; }
// Rename @...@ flags to $...$ flags and compile restriction.
@@ -3898,7 +4291,10 @@ HfstTransducer &HfstTransducer::remove_illegal_flag_paths(void)
HfstTransducer &HfstTransducer::lenient_composition( const HfstTransducer &another, bool /*harmonize*/)
{
-
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
if ( this->type != another.type )
{
HFST_THROW_MESSAGE(HfstTransducerTypeMismatchException, "HfstTransducer::lenient_composition");
@@ -3916,7 +4312,10 @@ HfstTransducer &HfstTransducer::lenient_composition( const HfstTransducer &anoth
HfstTransducer &HfstTransducer::cross_product( const HfstTransducer &another, bool /*harmonize*/)
{
-
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
if ( this->type != another.type )
{
HFST_THROW_MESSAGE(HfstTransducerTypeMismatchException, "HfstTransducer::cross_product");
@@ -3931,7 +4330,7 @@ HfstTransducer &HfstTransducer::cross_product( const HfstTransducer &another, bo
HfstTransducer t2_proj(automata2);
t2_proj.input_project();
- if ( not t1_proj.compare(automata1) || not t2_proj.compare(automata2) )
+ if ( ! t1_proj.compare(automata1) || ! t2_proj.compare(automata2) )
{
HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstTransducer::cross_product");
}
@@ -4053,6 +4452,10 @@ bool code_symbols_for_shuffle(const StringPair &sp, StringPairSet &sps)
HfstTransducer &HfstTransducer::shuffle(const HfstTransducer &another, bool)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
if (this->type != another.type)
HFST_THROW_MESSAGE(TransducerTypeMismatchException,
"HfstTransducer::shuffle(const HfstTransducer&)");
@@ -4141,6 +4544,10 @@ HfstTransducer &HfstTransducer::shuffle(const HfstTransducer &another, bool)
HfstTransducer &HfstTransducer::priority_union (const HfstTransducer &another, bool, bool encode_epsilons)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
if ( this->type != another.type )
{
HFST_THROW_MESSAGE(HfstTransducerTypeMismatchException,
@@ -4185,6 +4592,10 @@ HfstTransducer &HfstTransducer::priority_union (const HfstTransducer &another, b
HfstTransducer &HfstTransducer::compose_intersect
(const HfstTransducerVector &v, bool invert, bool)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
// Foma transducers don't harmonize porperly. If the input is foma
// transducers, convert to openfst type.
bool convert_to_openfst = false;
@@ -4368,6 +4779,9 @@ HfstTransducer &HfstTransducer::concatenate
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::concatenate,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::concatenate,
+#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//&hfst::implementations::MyTransducerLibraryTransducer::concatenate,
@@ -4455,6 +4869,9 @@ HfstTransducer &HfstTransducer::disjunct
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::disjunct,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::disjunct,
+#endif
/* Add here your implementation. */
const_cast<HfstTransducer&>(another), harmonize); }
@@ -4474,6 +4891,9 @@ HfstTransducer &HfstTransducer::intersect
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::intersect,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::intersect,
+#endif
/* Add here your implementation. */
const_cast<HfstTransducer&>(another), harmonize); }
@@ -4493,6 +4913,9 @@ HfstTransducer &HfstTransducer::subtract
#if HAVE_FOMA
&hfst::implementations::FomaTransducer::subtract,
#endif
+#if HAVE_XFSM
+ &hfst::implementations::XfsmTransducer::subtract,
+#endif
/* Add here your implementation. */
const_cast<HfstTransducer&>(another), harmonize); }
@@ -4654,7 +5077,7 @@ HfstTransducer &HfstTransducer::convert(const HfstTransducer &t,
}
if (type == t.type)
{ return *(new HfstTransducer(t)); }
- if (not is_implementation_type_available(type)) {
+ if (! is_implementation_type_available(type)) {
HFST_THROW_MESSAGE
(ImplementationTypeNotAvailableException,
"HfstTransducer::convert");
@@ -4684,6 +5107,10 @@ bool HfstTransducer::is_implementation_type_available
if (type == TROPICAL_OPENFST_TYPE || type == LOG_OPENFST_TYPE)
return false;
#endif
+#if !HAVE_XFSM
+ if (type == XFSM_TYPE)
+ return false;
+#endif
/* Add here your implementation. */
//#if !HAVE_MY_TRANSDUCER_LIBRARY
//if (type == MY_TRANSDUCER_LIBRARY_TYPE)
@@ -4696,7 +5123,7 @@ bool HfstTransducer::is_implementation_type_available
HfstTransducer &HfstTransducer::convert(ImplementationType type,
std::string options)
{
- if (not is_implementation_type_available(this->type)) {
+ if (! is_implementation_type_available(this->type)) {
HFST_THROW_MESSAGE(HfstFatalException,
"HfstTransducer::convert: the original type "
"of the transducer is not available!");
@@ -4708,7 +5135,7 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
"HfstTransducer::convert"); }
if (type == this->type)
{ return *this; }
- if (not is_implementation_type_available(type)) {
+ if (! is_implementation_type_available(type)) {
HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
"HfstTransducer::convert");
}
@@ -4724,6 +5151,14 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
foma_interface.delete_foma(implementation.foma);
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ internal =
+ ConversionFunctions::xfsm_to_hfst_basic_transducer
+ (implementation.xfsm);
+ //delete implementation.xfsm;
+ break;
+#endif
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//case MY_TRANSDUCER_LIBRARY_TYPE:
@@ -4820,6 +5255,13 @@ HfstTransducer &HfstTransducer::convert(ImplementationType type,
delete internal;
break;
#endif
+#if HAVE_XFSM
+ case XFSM_TYPE:
+ implementation.xfsm =
+ ConversionFunctions::hfst_basic_transducer_to_xfsm(internal);
+ delete internal;
+ break;
+#endif
case ERROR_TYPE:
default:
HFST_THROW(TransducerHasWrongTypeException);
@@ -4843,6 +5285,10 @@ void HfstTransducer::write_in_att_format
void HfstTransducer::write_in_att_format_number
(FILE * ofile, bool print_weights) const
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
hfst::implementations::HfstBasicTransducer net(*this);
net.write_in_att_format_number(ofile, print_weights);
}
@@ -4850,31 +5296,89 @@ void HfstTransducer::write_in_att_format_number
void HfstTransducer::write_in_att_format
(char * buffer, bool print_weights) const
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
hfst::implementations::HfstBasicTransducer net(*this);
net.write_in_att_format(buffer, print_weights);
}
void HfstTransducer::write_in_att_format(HfstFile &ofile, bool write_weights) const {
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
this->write_in_att_format(ofile.get_file(), write_weights);
}
void HfstTransducer::write_in_att_format
(FILE * ofile, bool print_weights) const
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
// Implemented only for internal transducer format.
hfst::implementations::HfstBasicTransducer net(*this);
net.write_in_att_format(ofile, print_weights);
}
+/* Implemented only for XFSM_TYPE. */
+void HfstTransducer::write_xfsm_transducer_in_att_format(const char * filename) const
+{
+ if (type != XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#if HAVE_XFSM
+ hfst::implementations::XfsmTransducer::write_in_att_format
+ (const_cast<NETptr>(this->implementation.xfsm), filename);
+#endif
+}
+
+/* Implemented only for XFSM_TYPE. */
+void HfstTransducer::write_xfsm_transducer_in_prolog_format(const char * filename) const
+{
+ if (type != XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#if HAVE_XFSM
+ hfst::implementations::XfsmTransducer::write_in_prolog_format
+ (const_cast<NETptr>(this->implementation.xfsm), filename);
+#endif
+}
+
+void HfstTransducer::write_in_prolog_format(FILE * file, const std::string & name,
+ bool write_weights)
+{
+ /* For big transducers, converting from xfsm is slow. */
+ if (type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+ HfstBasicTransducer fsm(*this);
+ fsm.write_in_prolog_format(file, name, write_weights);
+}
+
+HfstTransducer * HfstTransducer::prolog_file_to_xfsm_transducer(const char * filename)
+{
+#if HAVE_XFSM
+ HfstTransducer * retval = new HfstTransducer(hfst::XFSM_TYPE);
+ retval->implementation.xfsm = XfsmTransducer::prolog_file_to_xfsm_transducer(filename);
+ return retval;
+#else
+ HFST_THROW(FunctionNotImplementedException);
+#endif
+}
+
HfstTransducer::HfstTransducer(FILE * ifile,
ImplementationType type,
const std::string &epsilon_symbol):
type(type),anonymous(false),is_trie(false), name("")
{
-
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
unsigned int linecount=0;
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
"HfstTransducer::HfstTransducer"
"(FILE*, ImplementationType, const std::string&)");
@@ -4947,9 +5451,12 @@ HfstTransducer::HfstTransducer(FILE * ifile,
unsigned int & linecount):
type(type),anonymous(false),is_trie(false), name("")
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
-
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
"HfstTransducer::HfstTransducer"
"(FILE*, ImplementationType, const std::string&)");
@@ -5020,9 +5527,12 @@ HfstTransducer::HfstTransducer(HfstFile & ifile,
const std::string &epsilon_symbol):
type(type),anonymous(false),is_trie(false), name("")
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
-
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
"HfstTransducer::HfstTransducer"
"(FILE*, ImplementationType, const std::string&)");
@@ -5095,22 +5605,27 @@ HfstTransducer &HfstTransducer::read_in_att_format
(const std::string &filename, ImplementationType type,
const std::string &epsilon_symbol)
{
- FILE * ifile = fopen(filename.c_str(), "rb");
- if (ifile == NULL) {
+ if (type == XFSM_TYPE)
+ { HFST_THROW(FunctionNotImplementedException); }
+ FILE * ifile = fopen(filename.c_str(), "rb");
+ if (ifile == NULL) {
std::string message(filename);
HFST_THROW_MESSAGE(StreamNotReadableException, message);
- }
- HfstTokenizer::check_utf8_correctness(epsilon_symbol);
-
- HfstTransducer &retval = read_in_att_format(ifile, type, epsilon_symbol);
- fclose(ifile);
- return retval;
+ }
+ HfstTokenizer::check_utf8_correctness(epsilon_symbol);
+
+ HfstTransducer &retval = read_in_att_format(ifile, type, epsilon_symbol);
+ fclose(ifile);
+ return retval;
}
HfstTransducer &HfstTransducer::read_in_att_format
(FILE * ifile, ImplementationType type, const std::string &epsilon_symbol)
{
- if (not is_implementation_type_available(type))
+ if (type == XFSM_TYPE)
+ { HFST_THROW(FunctionNotImplementedException); }
+
+ if (! is_implementation_type_available(type))
HFST_THROW_MESSAGE(ImplementationTypeNotAvailableException,
"HfstTransducer::read_in_att_format");
@@ -5169,6 +5684,10 @@ HfstTransducer &HfstTransducer::assign(const HfstTransducer &another)
HfstTransducer &HfstTransducer::operator=(const HfstTransducer &another)
{
+#if HAVE_XFSM
+ if (this->type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
// Check for self-assignment.
if (&another == this)
{ return *this; }
@@ -5322,8 +5841,11 @@ HfstTransducer * HfstTransducer::read_lexc_ptr(const std::string &filename,
ImplementationType type,
bool verbose)
{
+ if (type == XFSM_TYPE)
+ { HFST_THROW(FunctionNotImplementedException); }
+
(void)filename;
- if (not is_implementation_type_available(type))
+ if (! is_implementation_type_available(type))
HFST_THROW(ImplementationTypeNotAvailableException);
HfstTransducer * retval = new HfstTransducer();
@@ -5332,10 +5854,12 @@ HfstTransducer * HfstTransducer::read_lexc_ptr(const std::string &filename,
{
#if HAVE_FOMA
case FOMA_TYPE:
+#if GENERATE_LEXC_WRAPPER
retval->implementation.foma = foma_interface.read_lexc(filename, verbose);
retval->type=FOMA_TYPE;
break;
-#endif
+#endif // GENERATE_LEXC_WRAPPER
+#endif // HAVE_FOMA
#if HAVE_SFST
case SFST_TYPE:
#endif
@@ -5395,6 +5919,10 @@ std::ostream & redirect(std::ostream &out, const HfstTransducer & t)
std::ostream & operator<<
(std::ostream &out, const HfstTransducer & t)
{
+#if HAVE_XFSM
+ if (t.type == XFSM_TYPE)
+ HFST_THROW(FunctionNotImplementedException);
+#endif
// Implemented only for internal transducer format.
hfst::implementations::HfstBasicTransducer net(t);
bool write_weights;
diff --git a/libhfst/src/HfstTransducer.h b/libhfst/src/HfstTransducer.h
index 8fad045..f27e5b4 100644
--- a/libhfst/src/HfstTransducer.h
+++ b/libhfst/src/HfstTransducer.h
@@ -36,6 +36,10 @@
#include "implementations/FomaTransducer.h"
#endif
+#if HAVE_XFSM
+#include "implementations/XfsmTransducer.h"
+#endif
+
/* Include here the header file of the files that take care
of the interaction between HFST and your transducer library. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
@@ -56,6 +60,8 @@
#include <map>
#include <set>
+#include "hfstdll.h"
+
/** @file HfstTransducer.h
\brief Declarations of HFST API functions and datatypes.
@@ -100,44 +106,61 @@ namespace hfst
using hfst::implementations::FomaTransducer;
#endif // #if HAVE_FOMA
+#if HAVE_XFSM
+ using hfst::implementations::XfsmTransducer;
+#endif // #if HAVE_XFSM
+
/* Add here the transducer class of your transducer library. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
// using hfst::implementations::MyTransducerLibraryTransducer;
//#endif // #if HAVE_MY_TRANSDUCER_LIBRARY
- HfstFile hfst_open(const char * filename, const char * mode);
- HfstFile hfst_stdout();
- HfstFile hfst_stdin();
+ HFSTDLL HfstFile hfst_open(const char * filename, const char * mode);
+ HFSTDLL HfstFile hfst_stdout();
+ HFSTDLL HfstFile hfst_stdin();
// *** TESTING AND OPTIMIZATION...
+#if HAVE_XFSM
+ HFSTDLL void initialize_xfsm();
+
+ class InitializeXfsm
+ {
+ public:
+ HFSTDLL InitializeXfsm();
+ };
+#endif
+
enum MinimizationAlgorithm { HOPCROFT, BRZOZOWSKI };
/* Which minimization algorithm is used.
In foma, Hopcroft is always used.
In OpenFst and SFST, the default algorithm is Hopcroft. */
- void set_minimization_algorithm(MinimizationAlgorithm);
- MinimizationAlgorithm get_minimization_algorithm();
+ HFSTDLL void set_minimization_algorithm(MinimizationAlgorithm);
+ HFSTDLL MinimizationAlgorithm get_minimization_algorithm();
+
+ HFSTDLL void set_encode_weights(bool);
+ HFSTDLL bool get_encode_weights();
- void set_encode_weights(bool);
- bool get_encode_weights();
+ HFSTDLL void set_minimize_even_if_already_minimal(bool);
+ HFSTDLL bool get_minimize_even_if_already_minimal();
- void set_xerox_composition(bool);
- bool get_xerox_composition();
+ HFSTDLL void set_xerox_composition(bool);
+ HFSTDLL bool get_xerox_composition();
- void set_flag_is_epsilon_in_composition(bool);
- bool get_flag_is_epsilon_in_composition();
+ HFSTDLL void set_flag_is_epsilon_in_composition(bool);
+ HFSTDLL bool get_flag_is_epsilon_in_composition();
/* Whether in harmonization the smaller transducer is always harmonized
according to the bigger transducer. */
- void set_harmonize_smaller(bool);
- bool get_harmonize_smaller();
+ HFSTDLL void set_harmonize_smaller(bool);
+ HFSTDLL bool get_harmonize_smaller();
/* Whether unknown and identity symbols are used. By default, they are used.
These symbols are always reserved for use and included in alphabets
of transducers, but code optimization is possible if it is known
that they do not appear in transducer transitions. */
- void set_unknown_symbols_in_use(bool);
- bool get_unknown_symbols_in_use();
+ HFSTDLL void set_unknown_symbols_in_use(bool);
+ HFSTDLL bool get_unknown_symbols_in_use();
// *** ...TESTING AND OPTIMIZATION
@@ -259,6 +282,9 @@ An example:
#if HAVE_FOMA
fsm * foma;
#endif
+#if HAVE_XFSM
+ NETptr xfsm;
+#endif
/* Add here your own transducer backend implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
@@ -290,6 +316,9 @@ An example:
static hfst::implementations::FomaTransducer foma_interface;
#endif
static hfst::implementations::HfstOlTransducer hfst_ol_interface;
+#if HAVE_XFSM
+ static hfst::implementations::XfsmTransducer xfsm_interface;
+#endif
/* Add here the class that takes care of the interaction between
HFST and your transducer library. */
@@ -340,12 +369,12 @@ An example:
public:
/* whether HFST is linked to the transducer library
needed by implementation type \a type. */
- static bool is_implementation_type_available(ImplementationType type);
+ HFSTDLL static bool is_implementation_type_available(ImplementationType type);
- unsigned int number_of_states() const;
- unsigned int number_of_arcs() const;
+ HFSTDLL unsigned int number_of_states() const;
+ HFSTDLL unsigned int number_of_arcs() const;
- void twosided_flag_diacritics();
+ HFSTDLL void twosided_flag_diacritics();
protected:
/* For internal use: Create a tokenizer that recognizes all symbols
@@ -404,10 +433,13 @@ An example:
(const std::string &filename, ImplementationType type,
const std::string &epsilon_symbol);
+ public:
+ HFSTDLL static HfstTransducer * prolog_file_to_xfsm_transducer(const char * filename);
+
/* For debugging */
public:
- void print_alphabet();
+ HFSTDLL void print_alphabet();
protected:
static float get_profile_seconds(ImplementationType type);
@@ -430,7 +462,7 @@ An example:
uninitialized. An uninitialized transducer is likely to cause a
TransducerHasWrongTypeException at some point unless it is given
a value at some point. */
- HfstTransducer();
+ HFSTDLL HfstTransducer();
/** \brief Create an empty transducer, i.e. a transducer that does not
recognize any string. The type of the transducer is defined by \a type.
@@ -438,7 +470,7 @@ An example:
@note Use HfstTransducer("@_EPSILON_SYMBOL_@")
to create an epsilon transducer.
**/
- HfstTransducer(ImplementationType type);
+ HFSTDLL HfstTransducer(ImplementationType type);
/** \brief Create a transducer by tokenizing the utf8 string \a utf8_string
with tokenizer \a multichar_symbol_tokenizer.
@@ -458,7 +490,7 @@ An example:
\endverbatim
@see HfstTokenizer **/
- HfstTransducer(const std::string& utf8_str,
+ HFSTDLL HfstTransducer(const std::string& utf8_str,
const HfstTokenizer &multichar_symbol_tokenizer,
ImplementationType type);
@@ -486,7 +518,7 @@ An example:
\endverbatim
@see HfstTokenizer **/
- HfstTransducer(const std::string& input_utf8_str,
+ HFSTDLL HfstTransducer(const std::string& input_utf8_str,
const std::string& output_utf8_str,
const HfstTokenizer &multichar_symbol_tokenizer,
ImplementationType type);
@@ -495,18 +527,18 @@ An example:
\a sps. The type of the transducer is defined by \a type. \a cyclic
defines whether the transducer recognizes any number (from zero to
infinity, inclusive) of consecutive string pairs in \s sps. */
- HfstTransducer(const StringPairSet & sps, ImplementationType type,
+ HFSTDLL HfstTransducer(const StringPairSet & sps, ImplementationType type,
bool cyclic=false);
/* \brief Create a transducer that recognizes the concatenation of
string pairs in \a spv. The type of the transducer is defined
by \a type. */
- HfstTransducer(const StringPairVector & spv, ImplementationType type);
+ HFSTDLL HfstTransducer(const StringPairVector & spv, ImplementationType type);
/* \brief Create a transducer that recognizes the concatenation of the
unions of string pairs in string pair sets in \a spsv. The type of
the transducer is defined by \a type. */
- HfstTransducer(const std::vector<StringPairSet> & spsv,
+ HFSTDLL HfstTransducer(const std::vector<StringPairSet> & spsv,
ImplementationType type);
/** \brief Read a binary transducer from transducer stream \a in.
@@ -529,29 +561,29 @@ An example:
@throws MissingOpenFstInputSymbolTableException
@see HfstInputStream **/
- HfstTransducer(HfstInputStream &in);
+ HFSTDLL HfstTransducer(HfstInputStream &in);
/** \brief Create a deep copy of transducer \a another. **/
- HfstTransducer(const HfstTransducer &another);
+ HFSTDLL HfstTransducer(const HfstTransducer &another);
/** \brief Create an HFST transducer equivalent to
HFST basic transducer \a t. The type of the created transducer
is defined by \a type. **/
- HfstTransducer(const hfst::implementations::HfstBasicTransducer &t,
+ HFSTDLL HfstTransducer(const hfst::implementations::HfstBasicTransducer &t,
ImplementationType type);
/** \brief Create a transducer that recognizes the string pair
<"symbol","symbol">, i.e. [symbol:symbol].
The type of the transducer is defined by \a type.
@see String **/
- HfstTransducer(const std::string &symbol, ImplementationType type);
+ HFSTDLL HfstTransducer(const std::string &symbol, ImplementationType type);
/** \brief Create a transducer that recognizes the string pair
<"isymbol","osymbol">, i.e [isymbol:osymbol].
The type of the transducer is defined by \a type.
@see String **/
- HfstTransducer(const std::string &isymbol, const std::string &osymbol,
- ImplementationType type);
+ HFSTDLL HfstTransducer(const std::string &isymbol, const std::string &osymbol,
+ ImplementationType type);
/** \brief Create a transducer of type \a type as defined in AT&T format
in FILE \a ifile. \a epsilon_symbol defines how epsilons
@@ -626,24 +658,24 @@ in \a ifile.
@see #write_in_att_format(FILE*,bool)const
@see String
**/
- HfstTransducer(FILE * ifile, ImplementationType type,
+ HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type,
const std::string &epsilon_symbol, unsigned int & linecount);
- HfstTransducer(FILE * ifile, ImplementationType type,
+ HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type,
const std::string &epsilon_symbol);
- HfstTransducer(HfstFile &ifile, ImplementationType type,
+ HFSTDLL HfstTransducer(HfstFile &ifile, ImplementationType type,
const std::string &epsilon_symbol);
/** \brief Destructor. **/
- virtual ~HfstTransducer(void);
+ HFSTDLL virtual ~HfstTransducer(void);
/** @brief Assign this transducer a new value equivalent to transducer
\a another. */
- HfstTransducer &operator=(const HfstTransducer &another);
+ HFSTDLL HfstTransducer &operator=(const HfstTransducer &another);
- HfstTransducer &assign(const HfstTransducer &another);
+ HFSTDLL HfstTransducer &assign(const HfstTransducer &another);
// ------------------------------------------------------------
// ----------- Properties, comparison, conversion -------------
@@ -651,11 +683,11 @@ in \a ifile.
/** \brief Rename the transducer \a name.
@see get_name */
- void set_name(const std::string &name);
+ HFSTDLL void set_name(const std::string &name);
/** \brief Get the name of the transducer.
@see set_name */
- std::string get_name() const;
+ HFSTDLL std::string get_name() const;
/**
* @brief Set arbitrary string property @a property to @a value.
@@ -666,26 +698,26 @@ in \a ifile.
* that does not follow this convention may affect the behavior of
* transducer in future releases.
*/
- void set_property(const std::string& property, const std::string& value);
+ HFSTDLL void set_property(const std::string& property, const std::string& value);
/**
* @brief Get arbitrary string propert @a property.
* get_property("name") works like get_name.
*/
- std::string get_property(const std::string& property) const;
+ HFSTDLL std::string get_property(const std::string& property) const;
/**
* @brief Get all properties form transducer.
*/
- const std::map<std::string,std::string>& get_properties() const;
+ HFSTDLL const std::map<std::string,std::string>& get_properties() const;
/** \brief Get the alphabet of the transducer.
The alphabet is defined as the set of symbols known
to the transducer. */
- StringSet get_alphabet() const;
+ HFSTDLL StringSet get_alphabet() const;
/** \brief Get first input level symbols of strings recognized
(or rejected, if they end in a non-final state) by the transducer. */
- StringSet get_first_input_symbols() const;
+ HFSTDLL StringSet get_first_input_symbols() const;
/** \brief Harmonize transducers this and another.
@@ -695,15 +727,15 @@ in \a ifile.
Then the unknown and identity symbols are expanded
in both transducers. If this and \a another have type FOMA_TYPE,
nothing is done, since foma takes care of harmonization. */
- void harmonize(HfstTransducer &another);
+ HFSTDLL void harmonize(HfstTransducer &another);
/** \brief Explicitly insert \a symbol to the alphabet
of the transducer.
@note Usually this function is not needed since new symbols are
added to the alphabet by default. */
- void insert_to_alphabet(const std::string &symbol);
- void insert_to_alphabet(const std::set<std::string> &symbols);
+ HFSTDLL void insert_to_alphabet(const std::string &symbol);
+ HFSTDLL void insert_to_alphabet(const std::set<std::string> &symbols);
/** \brief Remove \a symbol from the alphabet of the transducer.
CURRENTLY NOT IMPLEMENTED.
@@ -711,8 +743,12 @@ in \a ifile.
@pre \a symbol does not occur in any transition of the transducer.
@note Use with care, removing a symbol that occurs in a transition
of the transducer can have unexpected results. */
- void remove_from_alphabet(const std::string &symbol);
- void remove_from_alphabet(const std::set<std::string> &symbols);
+ HFSTDLL void remove_from_alphabet(const std::string &symbol);
+ HFSTDLL void remove_from_alphabet(const std::set<std::string> &symbols);
+
+ // For XFSM format
+ HFSTDLL void remove_symbols_from_alphabet(const StringSet & symbols);
+
/** @brief Remove all symbols that do not occur in transitions of
the transducer from its alphabet.
@@ -725,16 +761,16 @@ in \a ifile.
Epsilon, unknown and identity \link hfst::String symbols\endlink
are always included in the alphabet. */
- HfstTransducer &prune_alphabet(bool force=true);
+ HFSTDLL HfstTransducer &prune_alphabet(bool force=true);
/** \brief Whether the transducer is cyclic. */
- bool is_cyclic(void) const;
+ HFSTDLL bool is_cyclic(void) const;
/** \brief Whether the transducer is an automaton. */
- bool is_automaton(void) const;
+ HFSTDLL bool is_automaton(void) const;
/** \brief The implementation type of the transducer. */
- ImplementationType get_type(void) const;
+ HFSTDLL ImplementationType get_type(void) const;
/** \brief Whether this transducer and \a another are equivalent.
@@ -742,7 +778,7 @@ in \a ifile.
string pairs with the same weights
and the same alignments.
*/
- bool compare(const HfstTransducer &another, bool harmonize=true) const;
+ HFSTDLL bool compare(const HfstTransducer &another, bool harmonize=true) const;
/** \brief Convert the transducer into an equivalent transducer
in format \a type.
@@ -760,7 +796,7 @@ in \a ifile.
@note For conversion between implementations::HfstTransitionGraph and HfstTransducer,
see HfstTransducer(const hfst::implementations::HfstBasicTransducer&, ImplementationType) and #hfst::implementations::HfstTransitionGraph::HfstTransitionGraph(const hfst::HfstTransducer&).
*/
- HfstTransducer &convert(ImplementationType type, std::string options="");
+ HFSTDLL HfstTransducer &convert(ImplementationType type, std::string options="");
// --------------------------------------------------------
@@ -821,13 +857,21 @@ This will yield a file "testfile.att" that looks as follows:
@see operator<<(std::ostream &out, const HfstTransducer &t)
@see HfstTransducer(FILE*, ImplementationType, const std::string&) */
- void write_in_att_format(FILE * ofile, bool write_weights=true) const;
+ HFSTDLL void write_in_att_format(FILE * ofile, bool write_weights=true) const;
+
+ HFSTDLL void write_in_att_format(HfstFile & ofile, bool write_weights=true) const;
+
+ HFSTDLL void write_in_att_format(char * buffer, bool write_weights=true) const;
- void write_in_att_format(HfstFile & ofile, bool write_weights=true) const;
+ /* For XFSM_TYPE. */
+ HFSTDLL void write_xfsm_transducer_in_att_format(const char * filename) const;
+ HFSTDLL void write_xfsm_transducer_in_prolog_format(const char * filename) const;
- void write_in_att_format(char * buffer, bool write_weights=true) const;
+ /* For other types. */
+ HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
+ bool write_weights=true);
- void write_in_att_format_number
+ HFSTDLL void write_in_att_format_number
(FILE * ofile, bool write_weights=true) const;
@@ -839,8 +883,9 @@ This will yield a file "testfile.att" that looks as follows:
If the file does not exist, it is created.
@see #write_in_att_format */
- void write_in_att_format(const std::string &filename,
- bool write_weights=true) const;
+ HFSTDLL void write_in_att_format(const std::string &filename,
+ bool write_weights=true) const;
+
public:
/* \brief Call \a callback with some or all string pairs recognized
@@ -852,7 +897,7 @@ This will yield a file "testfile.att" that looks as follows:
indicating unlimited. Note that if the transducer is cyclic and
cycles aren't capped,
the search will not end until the callback returns false. */
- void extract_paths(ExtractStringsCb& callback, int cycles=-1) const;
+ HFSTDLL void extract_paths(ExtractStringsCb& callback, int cycles=-1) const;
/** \brief Extract a maximum of \a max_num paths that are
recognized by the transducer following a maximum of \a cycles cycles
@@ -935,31 +980,31 @@ ccc : ddd
@see #n_best
@see hfst::HfstTransducer::extract_paths_fd(hfst::HfstTwoLevelPaths&, int, int, bool) const
*/
- void extract_paths
+ HFSTDLL void extract_paths
(HfstTwoLevelPaths &results, int max_num=-1, int cycles=-1) const;
- void extract_random_paths
+ HFSTDLL void extract_random_paths
(HfstTwoLevelPaths &results, int max_num) const;
- void extract_random_paths_fd
+ HFSTDLL void extract_random_paths_fd
(HfstTwoLevelPaths &results, int max_num, bool filter_fd) const;
/* \brief Call \a callback with extracted strings that are not
invalidated by flag diacritic rules.
@see extract_paths(HfstTwoLevelPaths&, int, int) */
- void extract_paths_fd
+ HFSTDLL void extract_paths_fd
(ExtractStringsCb& callback, int cycles=-1, bool filter_fd=true) const;
// todo: handle flag diacritics
// todo: throw TransducerIsCyclicException, if cyclic
- void extract_shortest_paths
+ HFSTDLL void extract_shortest_paths
(HfstTwoLevelPaths &results) const;
- bool extract_longest_paths
+ HFSTDLL bool extract_longest_paths
(HfstTwoLevelPaths &results, bool obey_flags=true /*,bool show_flags=false*/) const;
- int longest_path_size(bool obey_flags=true) const;
+ HFSTDLL int longest_path_size(bool obey_flags=true) const;
public:
/** \brief Extract a maximum of \a max_num paths that are
@@ -995,7 +1040,7 @@ ccc : ddd
@bug Does not work for HFST_OL_TYPE or HFST_OLW_TYPE?
@throws TransducerIsCyclicException
@see extract_paths(HfstTwoLevelPaths&, int, int) const */
- void extract_paths_fd
+ HFSTDLL void extract_paths_fd
(HfstTwoLevelPaths &results, int max_num=-1, int cycles=-1,
bool filter_fd=true) const;
@@ -1009,7 +1054,7 @@ ccc : ddd
//! @todo Handle flag diacritics as ordinary symbols instead of calling
//! lookup_fd.
//! @sa lookup_fd
- HfstOneLevelPaths * lookup(const StringVector& s,
+ HFSTDLL HfstOneLevelPaths * lookup(const StringVector& s,
ssize_t limit = -1) const;
//! @brief Lookup or apply a single string \a s and
@@ -1017,7 +1062,7 @@ ccc : ddd
//!
//! This is an overloaded lookup function that leaves tokenizing to the
//! transducer.
- HfstOneLevelPaths * lookup(const std::string & s,
+ HFSTDLL HfstOneLevelPaths * lookup(const std::string & s,
ssize_t limit = -1) const;
//! @brief Lookup or apply a single string \a s minding flag diacritics
@@ -1049,7 +1094,7 @@ ccc : ddd
//!
//! @todo Do not ignore argument \a limit.
//!
- HfstOneLevelPaths * lookup_fd(const StringVector& s,
+ HFSTDLL HfstOneLevelPaths * lookup_fd(const StringVector& s,
ssize_t limit = -1) const;
//! @brief Lookup or apply a single string \a s minding flag diacritics
@@ -1066,7 +1111,7 @@ ccc : ddd
//!
//!
//!@sa lookup_fd
- HfstOneLevelPaths * lookup_fd(const std::string& s,
+ HFSTDLL HfstOneLevelPaths * lookup_fd(const std::string& s,
ssize_t limit = -1) const;
//! @brief Lookup or apply a single string \a s and store a maximum of
@@ -1077,7 +1122,7 @@ ccc : ddd
//! #lookup(const StringVector&, ssize_t) const
//! but lookup is not done using a string and a tokenizer instead of
//! a StringVector.
- HfstOneLevelPaths * lookup(const HfstTokenizer& tok,
+ HFSTDLL HfstOneLevelPaths * lookup(const HfstTokenizer& tok,
const std::string &s, ssize_t limit = -1) const;
//! @brief Lookup or apply a single string \a s minding flag diacritics
@@ -1088,7 +1133,7 @@ ccc : ddd
//! #lookup_fd(const StringVector&, ssize_t) const
//! but uses a tokenizer and a string instead of a StringVector.
//!
- HfstOneLevelPaths * lookup_fd(
+ HFSTDLL HfstOneLevelPaths * lookup_fd(
const HfstTokenizer& tok,
const std::string &s, ssize_t limit = -1) const;
@@ -1106,10 +1151,10 @@ ccc : ddd
//! may get stuck if infinitely ambiguous
//! @return output parameter to store unique results
//! @todo todo
- HfstOneLevelPaths * lookdown(const StringVector& s,
+ HFSTDLL HfstOneLevelPaths * lookdown(const StringVector& s,
ssize_t limit = -1) const;
- HfstOneLevelPaths * lookdown(const std::string& s,
+ HFSTDLL HfstOneLevelPaths * lookdown(const std::string& s,
ssize_t limit = -1) const;
//! @brief (Not implemented) Lookdown a single string minding
@@ -1120,10 +1165,10 @@ ccc : ddd
//!
//! @sa lookdown
//! @todo todo
- HfstOneLevelPaths * lookdown_fd(StringVector& s,
+ HFSTDLL HfstOneLevelPaths * lookdown_fd(StringVector& s,
ssize_t limit = -1) const;
- HfstOneLevelPaths * lookdown_fd(const std::string& s,
+ HFSTDLL HfstOneLevelPaths * lookdown_fd(const std::string& s,
ssize_t limit = -1) const;
//! @brief Whether lookup of path \a s will have infinite results.
@@ -1133,37 +1178,37 @@ ccc : ddd
//! i.e. the argument \a s is ignored.
//!
//! @see lookup(HfstOneLevelPaths&, const StringVector&, ssize_t) const
- bool is_lookup_infinitely_ambiguous(const StringVector & s) const;
- bool is_lookup_infinitely_ambiguous(const std::string & s) const;
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s) const;
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const std::string & s) const;
//! @brief (Not implemented) Whether lookdown of path \a s will have
//! infinite results.
//! @todo todo
- bool is_lookdown_infinitely_ambiguous(const StringVector& s) const;
+ HFSTDLL bool is_lookdown_infinitely_ambiguous(const StringVector& s) const;
- bool is_infinitely_ambiguous() const ;
+ HFSTDLL bool is_infinitely_ambiguous() const ;
// -------------------------------------------
// --------- Optimization operations ---------
// -------------------------------------------
- HfstTransducer &eliminate_flags();
- HfstTransducer &eliminate_flag(const std::string &flag);
+ HFSTDLL HfstTransducer &eliminate_flags();
+ HFSTDLL HfstTransducer &eliminate_flag(const std::string &flag);
/** \brief Remove all <i>epsilon:epsilon</i> transitions
from the transducer so that the transducer remains equivalent. */
- HfstTransducer &remove_epsilons();
+ HFSTDLL HfstTransducer &remove_epsilons();
/** \brief Make transducer coaccessible. */
- HfstTransducer &prune();
+ HFSTDLL HfstTransducer &prune();
/** \brief Determinize the transducer.
Determinizing a transducer yields an equivalent transducer that has
no state with two or more transitions whose input:output symbol
pairs are the same. */
- HfstTransducer &determinize();
+ HFSTDLL HfstTransducer &determinize();
/** \brief Minimize the transducer.
@@ -1172,7 +1217,7 @@ ccc : ddd
@bug OpenFst's minimization algorithm seems to add epsilon
transitions to weighted transducers? */
- HfstTransducer &minimize();
+ HFSTDLL HfstTransducer &minimize();
/** \brief Extract \a n best paths of the transducer.
@@ -1189,7 +1234,7 @@ ccc : ddd
#SFST_TYPE. If HFST is not linked to OpenFst library, an
ImplementationTypeNotAvailableException is thrown.
*/
- HfstTransducer &n_best(unsigned int n);
+ HFSTDLL HfstTransducer &n_best(unsigned int n);
// ------------------------------------------------
@@ -1198,60 +1243,60 @@ ccc : ddd
/** \brief A concatenation of N transducers where N is any number
from zero to infinity. */
- HfstTransducer &repeat_star();
+ HFSTDLL HfstTransducer &repeat_star();
/** \brief A concatenation of N transducers where N is any number
from one to infinity. */
- HfstTransducer &repeat_plus();
+ HFSTDLL HfstTransducer &repeat_plus();
/** \brief A concatenation of \a n transducers. */
- HfstTransducer &repeat_n(unsigned int n);
+ HFSTDLL HfstTransducer &repeat_n(unsigned int n);
/** \brief A concatenation of N transducers where N is any number
from zero to \a n, inclusive.*/
- HfstTransducer &repeat_n_minus(unsigned int n);
+ HFSTDLL HfstTransducer &repeat_n_minus(unsigned int n);
/** \brief A concatenation of N transducers where N is any number
from \a n to infinity, inclusive.*/
- HfstTransducer &repeat_n_plus(unsigned int n);
+ HFSTDLL HfstTransducer &repeat_n_plus(unsigned int n);
/** \brief A concatenation of N transducers where N is any number
from \a n to \a k, inclusive.*/
- HfstTransducer& repeat_n_to_k(unsigned int n, unsigned int k);
+ HFSTDLL HfstTransducer& repeat_n_to_k(unsigned int n, unsigned int k);
/** \brief Disjunct the transducer with an epsilon transducer. */
- HfstTransducer &optionalize();
+ HFSTDLL HfstTransducer &optionalize();
/** \brief Swap the input and output symbols of each transition
in the transducer. */
- HfstTransducer &invert();
+ HFSTDLL HfstTransducer &invert();
/** \brief Reverse the transducer.
A reverted transducer accepts the string "n(0) n(1) ... n(N)"
iff the original
transducer accepts the string "n(N) n(N-1) ... n(0)" */
- HfstTransducer &reverse();
+ HFSTDLL HfstTransducer &reverse();
/** \brief Extract the input language of the transducer.
All transition symbol pairs <i>isymbol:osymbol</i> are changed
to <i>isymbol:isymbol</i>. */
- HfstTransducer &input_project();
+ HFSTDLL HfstTransducer &input_project();
/** \brief Extract the output language of the transducer.
All transition symbol pairs <i>isymbol:osymbol</i> are changed
to <i>osymbol:osymbol</i>. */
- HfstTransducer &output_project();
+ HFSTDLL HfstTransducer &output_project();
/** \brief Compose this transducer with \a another. */
- HfstTransducer &compose(const HfstTransducer &another,
+ HFSTDLL HfstTransducer &compose(const HfstTransducer &another,
bool harmonize=true);
- HfstTransducer &merge(const HfstTransducer &another, const std::map<std::string, std::set<std::string> > & list_symbols);
+ HFSTDLL HfstTransducer &merge(const HfstTransducer &another, const std::map<std::string, std::set<std::string> > & list_symbols);
- HfstTransducer &merge(const HfstTransducer &another, const struct hfst::xre::XreConstructorArguments & args);
+ HFSTDLL HfstTransducer &merge(const HfstTransducer &another, const struct hfst::xre::XreConstructorArguments & args);
/** \brief Compose this transducer with the intersection of
transducers in \a v. If \a invert is true, then compose the
@@ -1263,14 +1308,14 @@ ccc : ddd
@pre The transducers in \a v are deterministic and epsilon-free.
*/
- HfstTransducer &compose_intersect(const HfstTransducerVector &v,
+ HFSTDLL HfstTransducer &compose_intersect(const HfstTransducerVector &v,
bool invert=false, bool harmonize=true);
/** \brief Concatenate this transducer with \a another. */
- HfstTransducer &concatenate(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &concatenate(const HfstTransducer &another, bool harmonize=true);
/** \brief Disjunct this transducer with \a another. */
- HfstTransducer &disjunct(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &disjunct(const HfstTransducer &another, bool harmonize=true);
/** \brief Make priority union of this transducer with \a another.
*
@@ -1295,13 +1340,13 @@ ccc : ddd
*
* For more information, read: www.fsmbook.com
* */
- HfstTransducer &priority_union(const HfstTransducer &another, bool harmonize=true, bool encode_epsilons=true);
+ HFSTDLL HfstTransducer &priority_union(const HfstTransducer &another, bool harmonize=true, bool encode_epsilons=true);
/** \brief Make lenient composition of this transducer with \a.
* A .O. B = [ A .o. B ] .P. A
*/
- HfstTransducer &lenient_composition(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &lenient_composition(const HfstTransducer &another, bool harmonize=true);
/** \brief Make cross product of this transducer with \a.
* It pairs every string of this with every string of \a.
@@ -1311,7 +1356,7 @@ ccc : ddd
* If strings are not the same length, epsilon padding will be added in the end of the shorter string.
*
*/
- HfstTransducer &cross_product(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &cross_product(const HfstTransducer &another, bool harmonize=true);
/*
@@ -1324,7 +1369,7 @@ ccc : ddd
* @pre Both transducers must be automata, i.e. map strings onto themselves.
*
*/
- HfstTransducer &shuffle(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &shuffle(const HfstTransducer &another, bool harmonize=true);
/** \brief Create universal pair transducer of \a type.
*
@@ -1333,7 +1378,7 @@ ccc : ddd
*
* Transducer weight is 0.
*/
- static HfstTransducer universal_pair ( ImplementationType type );
+ HFSTDLL static HfstTransducer universal_pair ( ImplementationType type );
/** \brief Create identity pair transducer of \a type.
*
@@ -1342,20 +1387,20 @@ ccc : ddd
*
* Transducer weight is 0.
*/
- static HfstTransducer identity_pair ( ImplementationType type );
+ HFSTDLL static HfstTransducer identity_pair ( ImplementationType type );
/* For HfstCompiler: Optimized disjunction function. */
- HfstTransducer &disjunct(const StringPairVector &spv);
+ HFSTDLL HfstTransducer &disjunct(const StringPairVector &spv);
/** \brief Intersect this transducer with \a another. */
- HfstTransducer &intersect(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &intersect(const HfstTransducer &another, bool harmonize=true);
/** \brief Subtract transducer \a another from this transducer. */
- HfstTransducer &subtract(const HfstTransducer &another, bool harmonize=true);
+ HFSTDLL HfstTransducer &subtract(const HfstTransducer &another, bool harmonize=true);
// ------------------------------------------------
@@ -1372,7 +1417,7 @@ ccc : ddd
transducer will be exapanded byt the symbols in symbol
pair. Otherwise they aren't.
*/
- HfstTransducer &insert_freely(const StringPair &symbol_pair, bool harmonize=true);
+ HFSTDLL HfstTransducer &insert_freely(const StringPair &symbol_pair, bool harmonize=true);
/** \brief Freely insert a copy of \a tr into the transducer.
@@ -1389,7 +1434,7 @@ ccc : ddd
Conversion is carried out for an HfstTransducer, if this function
is called.
*/
- HfstTransducer &insert_freely(const HfstTransducer &tr, bool harmonize=true);
+ HFSTDLL HfstTransducer &insert_freely(const HfstTransducer &tr, bool harmonize=true);
/** \brief Substitute all transition \a sp with transitions \a sps
as defined by function \a func.
@@ -1432,7 +1477,7 @@ t.substitute(&function);
@see String
*/
- HfstTransducer &substitute
+ HFSTDLL HfstTransducer &substitute
(bool (*func)(const StringPair &sp, StringPairSet &sps));
/** \brief Substitute all transition symbols equal to \a old_symbol
@@ -1450,7 +1495,7 @@ t.substitute(&function);
The transition weights remain the same.
@see String */
- HfstTransducer &substitute(const std::string &old_symbol,
+ HFSTDLL HfstTransducer &substitute(const std::string &old_symbol,
const std::string &new_symbol,
bool input_side=true,
bool output_side=true);
@@ -1468,7 +1513,7 @@ t.substitute(&function);
@see String
*/
- HfstTransducer &substitute(const StringPair &old_symbol_pair,
+ HFSTDLL HfstTransducer &substitute(const StringPair &old_symbol_pair,
const StringPair &new_symbol_pair);
/** \brief Substitute all transitions equal to \a old_symbol_pair
@@ -1483,7 +1528,7 @@ t.substitute(&function);
@see String
*/
- HfstTransducer &substitute(const StringPair &old_symbol_pair,
+ HFSTDLL HfstTransducer &substitute(const StringPair &old_symbol_pair,
const StringPairSet &new_symbol_pair_set);
/** \brief Substitute all transition symbols as defined in \a substitutions.
@@ -1495,9 +1540,9 @@ t.substitute(&function);
This function performs all substitutions at the same time, so it is
more efficient than calling substitute separately for each substitution.
*/
- HfstTransducer &substitute(const HfstSymbolSubstitutions &substitutions);
+ HFSTDLL HfstTransducer &substitute(const HfstSymbolSubstitutions &substitutions);
- HfstTransducer &substitute_symbols(const HfstSymbolSubstitutions &substitutions);
+ HFSTDLL HfstTransducer &substitute_symbols(const HfstSymbolSubstitutions &substitutions);
/** \brief Substitute all transition symbol pairs as defined in \a substitutions.
@@ -1509,9 +1554,9 @@ t.substitute(&function);
This function performs all substitutions at the same time, so it is
more efficient than calling substitute separately for each substitution.
*/
- HfstTransducer &substitute(const HfstSymbolPairSubstitutions &substitutions);
+ HFSTDLL HfstTransducer &substitute(const HfstSymbolPairSubstitutions &substitutions);
- HfstTransducer &substitute_symbol_pairs(const HfstSymbolPairSubstitutions &substitutions);
+ HFSTDLL HfstTransducer &substitute_symbol_pairs(const HfstSymbolPairSubstitutions &substitutions);
/** \brief Substitute all transitions equal to \a symbol_pair
with a copy of transducer \a transducer.
@@ -1529,7 +1574,7 @@ t.substitute(&function);
@see String
*/
- HfstTransducer &substitute(const StringPair &symbol_pair,
+ HFSTDLL HfstTransducer &substitute(const StringPair &symbol_pair,
HfstTransducer &transducer, bool harmonize=true);
// -----------------------------------------------
@@ -1543,7 +1588,7 @@ t.substitute(&function);
If the HfstTransducer is of unweighted type
(#SFST_TYPE or #FOMA_TYPE), nothing is done.
*/
- HfstTransducer &set_final_weights(float weight, bool increment=false);
+ HFSTDLL HfstTransducer &set_final_weights(float weight, bool increment=false);
/** \brief Transform all transition and state weights as defined
in \a func.
@@ -1567,7 +1612,7 @@ transducer.transform_weights(&func);
If the HfstTransducer is of unweighted type
(#SFST_TYPE or #FOMA_TYPE), nothing is done.
*/
- HfstTransducer &transform_weights(float (*func)(float));
+ HFSTDLL HfstTransducer &transform_weights(float (*func)(float));
/** \brief Push weights towards initial or final state(s)
as defined by \a type.
@@ -1576,16 +1621,16 @@ transducer.transform_weights(&func);
(#SFST_TYPE or #FOMA_TYPE), nothing is done.
@see PushType
*/
- HfstTransducer &push_weights(PushType type);
+ HFSTDLL HfstTransducer &push_weights(PushType type);
/** \brief Compile a lexc file in file \a filename into an HfstTransducer
of type \a type and return the transducer. */
- static HfstTransducer * read_lexc_ptr(const std::string &filename,
+ HFSTDLL static HfstTransducer * read_lexc_ptr(const std::string &filename,
ImplementationType type,
bool verbose);
- static HfstTransducer read_lexc(const std::string &filename,
+ HFSTDLL static HfstTransducer read_lexc(const std::string &filename,
ImplementationType type,
bool verbose);
@@ -1594,7 +1639,7 @@ transducer.transform_weights(&func);
/* For each flag diacritic fd that is included in the alphabet of
transducer \a another but not in the alphabet of this transducer,
insert freely a transition fd:fd in this transducer. */
- void insert_freely_missing_flags_from
+ HFSTDLL void insert_freely_missing_flags_from
(const HfstTransducer &another);
/*
@@ -1605,26 +1650,28 @@ transducer.transform_weights(&func);
If \a insert_renamed_flags is true, then the flags from \a this are
inserted freely in \a another and vice versa after replacing.
*/
- void harmonize_flag_diacritics(HfstTransducer &another,
+ HFSTDLL void harmonize_flag_diacritics(HfstTransducer &another,
bool insert_renamed_flags=true);
- void insert_missing_symbols_to_alphabet_from(const HfstTransducer &another, bool only_special_symbols=false);
+ HFSTDLL void insert_missing_symbols_to_alphabet_from(const HfstTransducer &another, bool only_special_symbols=false);
+
+ HFSTDLL StringSet insert_missing_diacritics_to_alphabet_from(const HfstTransducer &another);
- static bool is_special_symbol(const std::string & symbol);
+ HFSTDLL static bool is_special_symbol(const std::string & symbol);
/* Whether the alphabet of transducer \a another includes flag diacritics
that are not included in the alphabet of this transducer. */
- bool check_for_missing_flags_in(const HfstTransducer &another) const;
+ HFSTDLL bool check_for_missing_flags_in(const HfstTransducer &another) const;
/* Return true if \a this has flag diacritics in the alphabet. */
- bool has_flag_diacritics(void) const;
+ HFSTDLL bool has_flag_diacritics(void) const;
// *** Friends **** //
- friend std::ostream& operator<<(std::ostream &out, const HfstTransducer &t);
- friend std::ostream& redirect(std::ostream &out, const HfstTransducer &t);
+ HFSTDLL friend std::ostream& operator<<(std::ostream &out, const HfstTransducer &t);
+ HFSTDLL friend std::ostream& redirect(std::ostream &out, const HfstTransducer &t);
friend class HfstInputStream;
friend class HfstOutputStream;
friend class hfst::implementations::HfstTransitionGraph<class C>;
@@ -1641,9 +1688,9 @@ transducer.transform_weights(&func);
The same as
#hfst::HfstTransducer::write_in_att_format(FILE*, bool) const
with ostreams. Weights are written if the type of \a t is weighted. */
- std::ostream &operator<<(std::ostream &out,const HfstTransducer &t);
+ HFSTDLL std::ostream &operator<<(std::ostream &out,const HfstTransducer &t);
- std::ostream &redirect(std::ostream &out,const HfstTransducer &t);
+ HFSTDLL std::ostream &redirect(std::ostream &out,const HfstTransducer &t);
/** \brief A namespace for functions that create two-level, replace,
restriction and coercion rule transducers. */
@@ -1654,26 +1701,26 @@ transducer.transform_weights(&func);
enum TwolType {twol_right, twol_left, twol_both};
/* helping methods */
- HfstTransducer universal_fst
+ HFSTDLL HfstTransducer universal_fst
(const StringPairSet &alphabet, ImplementationType type);
- HfstTransducer negation_fst
+ HFSTDLL HfstTransducer negation_fst
(const HfstTransducer &t, const StringPairSet &alphabet);
- HfstTransducer replace
+ HFSTDLL HfstTransducer replace
(HfstTransducer &t, ReplaceType repl_type, bool optional,
StringPairSet &alphabet);
- HfstTransducer replace_transducer
+ HFSTDLL HfstTransducer replace_transducer
(HfstTransducer &t, std::string lm, std::string rm,
ReplaceType repl_type, StringPairSet &alphabet);
- HfstTransducer replace_context
+ HFSTDLL HfstTransducer replace_context
(HfstTransducer &t, std::string m1, std::string m2,
StringPairSet &alphabet);
- HfstTransducer replace_in_context
+ HFSTDLL HfstTransducer replace_in_context
(HfstTransducerPair &context, ReplaceType repl_type,
HfstTransducer &t, bool optional, StringPairSet &alphabet);
/* Used by hfst-calculate. */
- HfstTransducer restriction
+ HFSTDLL HfstTransducer restriction
(HfstTransducerPairVector &contexts, HfstTransducer &mapping,
StringPairSet &alphabet, TwolType twol_type, int direction );
@@ -1709,7 +1756,7 @@ alphabet = set(a, a:b, b, c, d, e, ...)
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>
*/
- HfstTransducer two_level_if(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer two_level_if(HfstTransducerPair &context,
StringPairSet &mappings,
StringPairSet &alphabet);
@@ -1728,7 +1775,7 @@ alphabet = set(a, a:b, b, c, d, e, ...)
@see #two_level_if
*/
- HfstTransducer two_level_only_if(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer two_level_only_if(HfstTransducerPair &context,
StringPairSet &mappings,
StringPairSet &alphabet);
@@ -1746,7 +1793,7 @@ alphabet = set(a, a:b, b, c, d, e, ...)
@see #two_level_if
*/
- HfstTransducer two_level_if_and_only_if(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer two_level_if_and_only_if(HfstTransducerPair &context,
StringPairSet &mappings,
StringPairSet &alphabet);
@@ -1801,7 +1848,7 @@ alphabet = set(a, b, c)
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>
*/
- HfstTransducer replace_up(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer replace_up(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1812,12 +1859,12 @@ alphabet = set(a, b, c)
@see replace_up
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer replace_down(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer replace_down(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
- HfstTransducer replace_down_karttunen(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer replace_down_karttunen(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1829,7 +1876,7 @@ SFST manual</a>. */
@see replace_up
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer replace_right(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer replace_right(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1841,7 +1888,7 @@ SFST manual</a>. */
@see replace_up
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer replace_left(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer replace_left(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1851,7 +1898,7 @@ SFST manual</a>. */
in every context.
@see replace_up */
- HfstTransducer replace_up(HfstTransducer &mapping,
+ HFSTDLL HfstTransducer replace_up(HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1859,7 +1906,7 @@ SFST manual</a>. */
but \a mapping is performed in every context.
@see replace_up */
- HfstTransducer replace_down(HfstTransducer &mapping,
+ HFSTDLL HfstTransducer replace_down(HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1871,7 +1918,7 @@ SFST manual</a>. */
* \a Mapping is performed in every context.
*
* @see replace_up */
- HfstTransducer left_replace_up( HfstTransducer &mapping,
+ HFSTDLL HfstTransducer left_replace_up( HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1881,7 +1928,7 @@ SFST manual</a>. */
* B <- A is the inversion of A -> B.
*
* @see replace_up */
- HfstTransducer left_replace_up( HfstTransducerPair &context,
+ HFSTDLL HfstTransducer left_replace_up( HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1889,7 +1936,7 @@ SFST manual</a>. */
* of the input language. However, matching is done on the output side of \a mapping
*
* @see replace_up */
- HfstTransducer left_replace_down(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer left_replace_down(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1898,7 +1945,7 @@ SFST manual</a>. */
* of the input language. However, matching is done on the output side of \a mapping
*
* @see replace_up */
- HfstTransducer left_replace_down_karttunen( HfstTransducerPair &context,
+ HFSTDLL HfstTransducer left_replace_down_karttunen( HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1908,7 +1955,7 @@ SFST manual</a>. */
* and right context on the output side of \a mapping.
*
* @see replace_up */
- HfstTransducer left_replace_left(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer left_replace_left(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1918,7 +1965,7 @@ SFST manual</a>. */
* and right context on the input side of \a mapping.
*
* @see replace_up */
- HfstTransducer left_replace_right(HfstTransducerPair &context,
+ HFSTDLL HfstTransducer left_replace_right(HfstTransducerPair &context,
HfstTransducer &mapping,
bool optional,
StringPairSet &alphabet);
@@ -1938,7 +1985,7 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer restriction(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer restriction(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -1951,7 +1998,7 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer coercion(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer coercion(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -1967,7 +2014,7 @@ SFST manual</a>. */
#coercion
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a> */
- HfstTransducer restriction_and_coercion(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer restriction_and_coercion(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -1983,7 +2030,7 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer surface_restriction(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer surface_restriction(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -1999,7 +2046,7 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer surface_coercion(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer surface_coercion(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -2009,7 +2056,7 @@ SFST manual</a>. */
@see #surface_restriction #surface_coercion
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer surface_restriction_and_coercion
+ HFSTDLL HfstTransducer surface_restriction_and_coercion
(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -2025,7 +2072,7 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer deep_restriction(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer deep_restriction(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -2040,7 +2087,7 @@ SFST manual</a>. */
@see
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer deep_coercion(HfstTransducerPairVector &contexts,
+ HFSTDLL HfstTransducer deep_coercion(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
@@ -2050,7 +2097,7 @@ SFST manual</a>. */
@see #deep_restriction #deep_coercion
<a href="ftp://ftp.ims.uni-stuttgart.de/pub/corpora/SFST/SFST-Manual.pdf">
SFST manual</a>. */
- HfstTransducer deep_restriction_and_coercion
+ HFSTDLL HfstTransducer deep_restriction_and_coercion
(HfstTransducerPairVector &contexts,
HfstTransducer &mapping,
StringPairSet &alphabet);
diff --git a/libhfst/src/HfstXeroxRules.cc b/libhfst/src/HfstXeroxRules.cc
index 68db331..00d0966 100644
--- a/libhfst/src/HfstXeroxRules.cc
+++ b/libhfst/src/HfstXeroxRules.cc
@@ -2952,7 +2952,7 @@ namespace hfst
HfstTransducer c_proj2(_center);
c_proj2.output_project();
- if ( not c_proj1.compare(_center) || not c_proj2.compare(_center) )
+ if ( ! c_proj1.compare(_center) || ! c_proj2.compare(_center) )
{
HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
}
@@ -3059,8 +3059,8 @@ namespace hfst
HfstTransducer r_proj2(right);
r_proj2.output_project();
- if ( not l_proj1.compare(left) || not l_proj2.compare(left)
- || not r_proj1.compare(right) || not r_proj2.compare(right) )
+ if ( ! l_proj1.compare(left) || ! l_proj2.compare(left)
+ || ! r_proj1.compare(right) || ! r_proj2.compare(right) )
{
HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
}
@@ -3097,8 +3097,8 @@ namespace hfst
HfstTransducer r_proj2(right);
r_proj2.output_project();
- if ( not l_proj1.compare(left) || not l_proj2.compare(left)
- || not r_proj1.compare(right) || not r_proj2.compare(right) )
+ if ( ! l_proj1.compare(left) || ! l_proj2.compare(left)
+ || ! r_proj1.compare(right) || ! r_proj2.compare(right) )
{
HFST_THROW_MESSAGE(TransducersAreNotAutomataException, "HfstXeroxRules::restriction");
}
diff --git a/libhfst/src/Makefile.am b/libhfst/src/Makefile.am
index 3da3c4e..83f14c2 100644
--- a/libhfst/src/Makefile.am
+++ b/libhfst/src/Makefile.am
@@ -56,6 +56,11 @@ else
libhfst_la_LIBADD += $(top_builddir)/back-ends/openfst/src/lib/libfst.la
endif
+if WINDOWS
+ libhfst_la_LIBADD += -ldl
+endif
+
+
# we only have one library target but would like a particular includedir
# structure for various headers, so we do this
hfstincludedir = $(includedir)/hfst
@@ -91,13 +96,14 @@ HFST_HDRS = \
HfstOutputStream.h \
HfstXeroxRules.h \
HfstLookupFlagDiacritics.h \
- parsers/LexcCompiler.h parsers/XreCompiler.h parsers/PmatchCompiler.h
+ parsers/LexcCompiler.h parsers/XreCompiler.h parsers/PmatchCompiler.h \
+ hfstdll.h
### Add your library here ###
# implementations/MyTransducerLibraryTransducer.h
hfstinclude_HEADERS = $(HFST_HDRS)
-libhfst_la_LDFLAGS = -no-undefined -version-info 40:0:0
+libhfst_la_LDFLAGS = -no-undefined -version-info 41:0:0
LIBHFST_TSTS=HfstApply HfstInputStream HfstTransducer \
HfstOutputStream HfstXeroxRules HfstRules HfstSymbolDefs \
diff --git a/libhfst/src/hfst_apply_schemas.h b/libhfst/src/hfst_apply_schemas.h
index 4aacf67..c782947 100644
--- a/libhfst/src/hfst_apply_schemas.h
+++ b/libhfst/src/hfst_apply_schemas.h
@@ -27,6 +27,9 @@ HfstTransducer &apply(
#if HAVE_FOMA
fsm * (*foma_funct)(fsm *),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr),
+#endif
/* Add your library here */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)(my_namespace::MyFst *),
@@ -50,6 +53,9 @@ HfstTransducer &apply(
#if HAVE_FOMA
fsm * (*foma_funct)(fsm *, unsigned int),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr, unsigned int),
+#endif
/* Add your library here */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)
@@ -73,6 +79,9 @@ HfstTransducer &apply(
#if HAVE_FOMA
fsm * (*foma_funct)(fsm *, String, String),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr, String, String),
+#endif
/* Add your library here */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)
@@ -98,6 +107,9 @@ HfstTransducer &apply(
fsm * (*foma_funct)(fsm *,
fsm *),
#endif
+#if HAVE_XFSM
+ NETptr (*xfsm_funct)(NETptr, NETptr),
+#endif
/* Add your library here */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//my_namespace::MyFst * (*my_transducer_library_funct)
diff --git a/libhfst/src/hfstdll.h b/libhfst/src/hfstdll.h
new file mode 100644
index 0000000..82d2b0a
--- /dev/null
+++ b/libhfst/src/hfstdll.h
@@ -0,0 +1,12 @@
+#ifndef _HFSTDLL_H_
+#define _HFSTDLL_H_
+#ifdef _MSC_VER
+#ifdef HFSTEXPORT
+#define HFSTDLL __declspec(dllexport)
+#else
+#define HFSTDLL __declspec(dllimport)
+#endif // HFSTEXPORT
+#else
+#define HFSTDLL
+#endif // _MSC_VER
+#endif // _HFSTDLL_H_
diff --git a/libhfst/src/implementations/ConvertFomaTransducer.cc b/libhfst/src/implementations/ConvertFomaTransducer.cc
index 9b16ba3..b550a6c 100644
--- a/libhfst/src/implementations/ConvertFomaTransducer.cc
+++ b/libhfst/src/implementations/ConvertFomaTransducer.cc
@@ -67,7 +67,7 @@ namespace hfst { namespace implementations
bool &start_state_found)
{
// If the start state has not yet been encountered.
- if (not start_state_found) {
+ if (! start_state_found) {
start_state_id = (fsm)->state_no; // define the start state
start_state_found=true; // define that it is found
}
@@ -191,7 +191,7 @@ namespace hfst { namespace implementations
}
// If there was not an initial state in foma transducer,
- if (not start_state_found) {
+ if (! start_state_found) {
copy_alphabet(t, net);
return net; // we assume that the transducer is empty.
// instead of throwing an exception.
diff --git a/libhfst/src/implementations/ConvertOlTransducer.cc b/libhfst/src/implementations/ConvertOlTransducer.cc
index 365d8ef..2fe3217 100644
--- a/libhfst/src/implementations/ConvertOlTransducer.cc
+++ b/libhfst/src/implementations/ConvertOlTransducer.cc
@@ -228,7 +228,8 @@ void get_states_and_symbols(
it != flag_diacritics->end(); ++it) {
if (!is_epsilon(*it)) {
string_symbol_map[*it] = symbol_table.size();
- flag_symbols.insert(symbol_table.size());
+ // TODO: cl.exe: conversion from 'size_t' to 'char16_t'
+ flag_symbols.insert((unsigned short)symbol_table.size());
symbol_table.push_back(*it);
// don't increment seen_input_symbols - we use it for
// indexing
@@ -238,7 +239,7 @@ void get_states_and_symbols(
// 4) non-input symbols
for (std::set<std::string>::iterator it = other_symbols->begin();
it != other_symbols->end(); ++it) {
- if (!is_epsilon(*it) and input_symbols->count(*it) == 0 and
+ if (!is_epsilon(*it) && input_symbols->count(*it) == 0 &&
flag_diacritics->count(*it) == 0) {
string_symbol_map[*it] = symbol_table.size();
symbol_table.push_back(*it);
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.cc b/libhfst/src/implementations/ConvertTransducerFormat.cc
index b69eb5a..186cfc2 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.cc
+++ b/libhfst/src/implementations/ConvertTransducerFormat.cc
@@ -92,6 +92,12 @@ namespace hfst { namespace implementations
if (t.type == FOMA_TYPE)
return foma_to_hfst_basic_transducer(t.implementation.foma);
#endif // HAVE_FOMA
+
+#if HAVE_XFSM
+ if (t.type == XFSM_TYPE)
+ return xfsm_to_hfst_basic_transducer(t.implementation.xfsm);
+#endif // HAVE_FOMA
+
/* Add here your implementation. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.h b/libhfst/src/implementations/ConvertTransducerFormat.h
index 7dea891..b1ed34d 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.h
+++ b/libhfst/src/implementations/ConvertTransducerFormat.h
@@ -23,11 +23,11 @@
#include <map>
#if HAVE_OPENFST
-#ifdef WINDOWS
+#ifdef _MSC_VER
#include "back-ends/openfstwin/src/include/fst/fstlib.h"
#else
#include "back-ends/openfst/src/include/fst/fstlib.h"
-#endif // WINDOWS
+#endif // _MSC_VER
#endif // HAVE_OPENFST
#if HAVE_SFST
@@ -37,12 +37,14 @@
#if HAVE_FOMA
#ifndef _FOMALIB_H_
#define _FOMALIB_H_
-#include <stdbool.h>
-#define _Bool bool
#include "back-ends/foma/fomalib.h"
#endif // _FOMALIB_H_
#endif // HAVE_FOMA
+#if HAVE_XFSM
+#include "xfsm/xfsm_api.h"
+#endif
+
/* Add here your transducer library header (and possibly a guard). */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//#ifndef _MY_TRANSDUCER_LIBRARY_LIB_H_
@@ -161,6 +163,11 @@ namespace implementations {
(const HfstConstantTransducer * t); */
#endif // HAVE_FOMA
+#if HAVE_XFSM
+ static HfstBasicTransducer * xfsm_to_hfst_basic_transducer(NETptr t);
+ static NETptr hfst_basic_transducer_to_xfsm(const HfstBasicTransducer * t);
+#endif // HAVE_XFSM
+
#if HAVE_OPENFST
static HfstBasicTransducer * tropical_ofst_to_hfst_basic_transducer
(fst::StdVectorFst * t, bool has_hfst_header=true);
diff --git a/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc b/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc
index 1453b38..88f69dc 100644
--- a/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc
+++ b/libhfst/src/implementations/ConvertTropicalWeightTransducer.cc
@@ -46,7 +46,7 @@ namespace hfst { namespace implementations
if (inputsym != NULL) {
for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(inputsym));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
@@ -57,10 +57,10 @@ namespace hfst { namespace implementations
symbol table. If the transducer is an HFST tropical transducer, it
can have an output symbol table, but it is equivalent to the
input symbol table. */
- if (not has_hfst_header && outputsym != NULL) {
+ if (! has_hfst_header && outputsym != NULL) {
for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(outputsym));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
net->add_symbol_to_alphabet( it.Symbol() );
@@ -89,7 +89,7 @@ namespace hfst { namespace implementations
{
for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(inputsym));
- not it.Done(); it.Next() )
+ ! it.Done(); it.Next() )
{
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
@@ -100,7 +100,7 @@ namespace hfst { namespace implementations
{
for ( fst::SymbolTableIterator it =
fst::SymbolTableIterator(*(outputsym));
- not it.Done(); it.Next() )
+ ! it.Done(); it.Next() )
{
assert(it.Symbol() != "");
if (it.Value() != 0) // epsilon is not inserted
@@ -139,7 +139,7 @@ namespace hfst { namespace implementations
/* Go through all states */
for (fst::StateIterator<fst::StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
@@ -230,7 +230,7 @@ namespace hfst { namespace implementations
for (HfstBasicTransducer::HfstTransitionGraphAlphabet::iterator it
= net->alphabet.begin();
it != net->alphabet.end(); it++) {
- assert(not it->empty());
+ assert(! it->empty());
st.AddSymbol(*it, net->get_symbol_number(*it));
}
diff --git a/libhfst/src/implementations/ConvertXfsmTransducer.cc b/libhfst/src/implementations/ConvertXfsmTransducer.cc
new file mode 100644
index 0000000..d1afc6e
--- /dev/null
+++ b/libhfst/src/implementations/ConvertXfsmTransducer.cc
@@ -0,0 +1,252 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include "ConvertTransducerFormat.h"
+#include "HfstTransitionGraph.h"
+#include "HfstTransducer.h"
+
+#ifndef MAIN_TEST
+namespace hfst { namespace implementations
+{
+
+ /* -----------------------------------------------------------
+
+ Conversion functions between HfstBasicTransducer and xfsm transducer.
+
+ ---------------------------------------------------------- */
+
+
+#if HAVE_XFSM
+
+#include "XfsmTransducer.h"
+
+ // Insert all symbols in an xfsm transducer alphabet (sigma) into
+ // the alphabet of an HfstBasicTransducer.
+ static void copy_xfsm_alphabet_into_hfst_alphabet(NETptr t, HfstBasicTransducer * fsm)
+ {
+ ALPHABETptr alpha_ptr = net_sigma(t);
+ ALPH_ITptr alpha_it_ptr = start_alph_iterator(NULL, alpha_ptr);
+ id_type label_id = next_alph_id(alpha_it_ptr);
+
+ while(label_id != ID_NO_SYMBOL)
+ {
+ std::string symbol = XfsmTransducer::xfsm_symbol_to_hfst_symbol(label_id);
+ fsm->add_symbol_to_alphabet(symbol);
+ label_id = next_alph_id(alpha_it_ptr);
+ }
+ }
+
+
+ /* ----------------------------------------------------------------------
+
+ Create an HfstBasicTransducer equivalent to foma transducer \a t.
+
+ ---------------------------------------------------------------------- */
+
+ HfstBasicTransducer * ConversionFunctions::
+ xfsm_to_hfst_basic_transducer(NETptr t)
+ {
+ HfstBasicTransducer * result = new HfstBasicTransducer();
+
+ // Map states of t into states in result
+ std::map<STATEptr, HfstState> xfsm_to_hfst_state;
+
+ STATEptr state_ptr = t->body.states;
+ STATEptr start_ptr = t->start.state;
+
+ // Create states in result
+ while (state_ptr != NULL)
+ {
+ // initial state exists already
+ if (state_ptr != start_ptr)
+ {
+ (void)result->add_state();
+ }
+ state_ptr = state_ptr->next;
+ }
+ state_ptr = t->body.states;
+
+ // For each state in t, map the state into a state in result and make it final,
+ // if needed. States of t are stored in a stack, so we start numbering states of
+ // result from the biggest state number.
+ HfstState result_state = result->get_max_state();
+ while (state_ptr != NULL)
+ {
+ if (state_ptr == start_ptr)
+ {
+ // initial state exists already in result
+ xfsm_to_hfst_state.insert(std::pair<STATEptr, HfstState>(state_ptr, 0));
+ if (state_ptr->final != 0)
+ {
+ result->set_final_weight(0, 0);
+ }
+ }
+ else
+ {
+ xfsm_to_hfst_state.insert(std::pair<STATEptr, HfstState>(state_ptr, result_state));
+ if (state_ptr->final != 0)
+ {
+ result->set_final_weight(result_state, 0);
+ }
+ --result_state;
+ }
+ state_ptr = state_ptr->next;
+ }
+
+ state_ptr = t->body.states;
+
+ // For each state in t, go through its transitions and copy them into result.
+ while (state_ptr != NULL)
+ {
+ ARCptr arc_ptr = state_ptr->arc.set;
+ while (arc_ptr != NULL)
+ {
+ id_type label_id = arc_ptr->label;
+ std::string isymbol, osymbol;
+ XfsmTransducer::label_id_to_symbol_pair(label_id, isymbol, osymbol);
+
+ STATEptr target_state_ptr = arc_ptr->destination;
+
+ HfstBasicTransition tr(xfsm_to_hfst_state[target_state_ptr], isymbol, osymbol, 0);
+ result->add_transition(xfsm_to_hfst_state[state_ptr], tr);
+
+ arc_ptr = arc_ptr->next;
+ }
+ state_ptr = state_ptr->next;
+ }
+
+ // Copy alphabet of t into result.
+ copy_xfsm_alphabet_into_hfst_alphabet(t, result);
+
+ return result;
+ }
+
+ /* ------------------------------------------------------------------------
+
+ Create an xfsm transducer equivalent to HfstBasicTransducer \a hfst_fsm.
+
+ ------------------------------------------------------------------------ */
+
+ NETptr ConversionFunctions::
+ hfst_basic_transducer_to_xfsm(const HfstBasicTransducer * hfst_fsm)
+ {
+ NETptr result = null_net();
+
+ // Maps HfstBasicTransducer states (i.e. vector indices) into xfsm transducer states.
+ std::vector<STATEptr> state_vector;
+
+ // ---- Copy states -----
+ unsigned int fsm_state = 0;
+ for (HfstBasicTransducer::const_iterator it = hfst_fsm->begin();
+ it != hfst_fsm->end(); it++)
+ {
+ // 'null_net()' creates the initial state
+ if (fsm_state != 0)
+ {
+ // TODO: slow to call is_final_state each time a new state is added
+ STATEptr xfsm_state = add_state_to_net(result, hfst_fsm->is_final_state(fsm_state) ? 1 : 0);
+ state_vector.push_back(xfsm_state);
+ }
+ else
+ {
+ // Finality of the initial state is checked later.
+ state_vector.push_back(result->start.state);
+ }
+ fsm_state++;
+ }
+
+ // ----- Go through all states -----
+ unsigned int source_state=0;
+ for (HfstBasicTransducer::const_iterator it = hfst_fsm->begin();
+ it != hfst_fsm->end(); it++)
+ {
+ STATEptr xfsm_source_state = state_vector.at(source_state);
+ // ----- Go through the set of transitions in each state -----
+ for (HfstBasicTransducer::HfstTransitions::const_iterator tr_it
+ = it->begin();
+ tr_it != it->end(); tr_it++)
+ {
+ // Copy the transition
+ std::string isymbol = tr_it->get_input_symbol();
+ std::string osymbol = tr_it->get_output_symbol();
+ HfstState target_state = tr_it->get_target_state();
+
+ id_type ti = XfsmTransducer::symbol_pair_to_label_id(isymbol, osymbol);
+
+ if (isymbol == hfst::internal_identity)
+ {
+ if (osymbol != hfst::internal_identity)
+ throw "identity symbol cannot be on one side only";
+ // atomic OTHER label
+ ti = OTHER;
+ }
+ else
+ {
+ id_type input_id = XfsmTransducer::hfst_symbol_to_xfsm_symbol(isymbol);
+ id_type output_id = XfsmTransducer::hfst_symbol_to_xfsm_symbol(osymbol);
+ ti = id_pair_to_id(input_id, output_id);
+ }
+
+ STATEptr xfsm_target_state = state_vector.at(target_state);
+
+ if( add_arc_to_state(result, xfsm_source_state, ti, xfsm_target_state, NULL, 0) == NULL )
+ throw "add_arc_to_state failed";
+
+ }
+ // ----- transitions gone through -----
+ source_state++;
+ }
+ // ----- all states gone through -----
+
+ // If the initial state is final, make the result optional.
+ if (hfst_fsm->is_final_state(0))
+ {
+ result = optional_net(result, 0);
+ }
+
+ // Copy alphabet
+ ALPHABETptr ap = net_sigma(result);
+ const HfstBasicTransducer::HfstTransitionGraphAlphabet & alpha
+ = hfst_fsm->get_alphabet();
+ for (HfstBasicTransducer::HfstTransitionGraphAlphabet::iterator it
+ = alpha.begin();
+ it != alpha.end(); it++)
+ {
+ if (hfst::is_epsilon(*it) || hfst::is_unknown(*it) || hfst::is_identity(*it))
+ continue;
+ (void) alph_add_to(ap, XfsmTransducer::hfst_symbol_to_xfsm_symbol(it->c_str()), DONT_KEEP);
+ }
+
+ return result;
+ }
+
+#endif // HAVE_XFSM
+
+ }}
+#else // MAIN_TEST was defined
+
+#include <iostream>
+
+int main(int argc, char * argv[])
+{
+ std::cout << "Unit tests for " __FILE__ ":" << std::endl;
+
+ std::cout << "ok" << std::endl;
+ return 0;
+}
+
+#endif // MAIN_TEST
+
diff --git a/libhfst/src/implementations/FomaTransducer.cc b/libhfst/src/implementations/FomaTransducer.cc
index 5a3417e..585a7fb 100644
--- a/libhfst/src/implementations/FomaTransducer.cc
+++ b/libhfst/src/implementations/FomaTransducer.cc
@@ -77,7 +77,7 @@ namespace hfst { namespace implementations {
bool FomaInputStream::is_good(void)
{
- return not is_bad();
+ return ! is_bad();
};
bool FomaInputStream::is_fst(void)
@@ -525,7 +525,7 @@ namespace hfst { namespace implementations {
}
}
- hfst::HfstTwoLevelPath path(0, spv);
+ hfst::HfstTwoLevelPath path(float(0), spv);
hfst::ExtractStringsCb::RetVal ret = callback(path, final);
if(!ret.continueSearch || !ret.continuePath)
{
@@ -671,7 +671,7 @@ namespace hfst { namespace implementations {
}
if (final_initial) {
StringPairVector empty_spv;
- HfstTwoLevelPath epsilon_path(0, empty_spv);
+ HfstTwoLevelPath epsilon_path(float(0), empty_spv);
callback(epsilon_path, true /* final*/);
}
}
@@ -798,6 +798,7 @@ namespace hfst { namespace implementations {
return table;
}
+#if GENERATE_LEXC_WRAPPER
fsm * FomaTransducer::read_lexc(const std::string &filename, bool verbose)
{
char * filename_ = strdup(filename.c_str());
@@ -822,6 +823,7 @@ namespace hfst { namespace implementations {
free(lexcfile);
return retval;
}
+#endif
struct fsm * FomaTransducer::eliminate_flags(struct fsm * t)
{
@@ -849,7 +851,7 @@ namespace hfst { namespace implementations {
/* Read foma transducer . */
struct fsm * FomaTransducer::read_net(FILE *infile) {
- unsigned int READ_BUF_SIZE=4096;
+ const unsigned int READ_BUF_SIZE=4096;
char buf[READ_BUF_SIZE];
struct fsm *net;
struct fsm_state *fsm;
diff --git a/libhfst/src/implementations/FomaTransducer.h b/libhfst/src/implementations/FomaTransducer.h
index 16161bf..757f931 100644
--- a/libhfst/src/implementations/FomaTransducer.h
+++ b/libhfst/src/implementations/FomaTransducer.h
@@ -17,18 +17,13 @@
#include "HfstExceptionDefs.h"
#include "HfstExtractStrings.h"
#include "HfstFlagDiacritics.h"
-#include <stdbool.h> // foma uses _Bool
#include <stdlib.h>
#ifndef _FOMALIB_H_
#define _FOMALIB_H_
-#define _Bool bool
#include "back-ends/foma/fomalib.h"
#endif
-#ifndef WINDOWS
- #include <zlib.h>
-#endif
#include <cstdio>
#include <string>
#include <sstream>
@@ -159,7 +154,9 @@ namespace hfst {
static void delete_foma(fsm * net);
static void print_test(fsm * t);
+#if GENERATE_LEXC_WRAPPER
static fsm * read_lexc(const std::string &filename, bool verbose);
+#endif
static unsigned int number_of_states(fsm * net);
static unsigned int number_of_arcs(fsm * net);
diff --git a/libhfst/src/implementations/HfstTransition.h b/libhfst/src/implementations/HfstTransition.h
index 3f162cb..558bc9a 100644
--- a/libhfst/src/implementations/HfstTransition.h
+++ b/libhfst/src/implementations/HfstTransition.h
@@ -6,6 +6,8 @@
#include "../HfstDataTypes.h"
+#include "../hfstdll.h"
+
namespace hfst {
namespace implementations {
@@ -13,102 +15,102 @@ namespace hfst {
/** @brief A transition that consists of a target state and
transition data represented by class C.
- The easiest way to use this template is to choose the
- the implementation #HfstBasicTransition which is compatible with
- #HfstBasicTransducer.
-
- @see HfstBasicTransition
+ The easiest way to use this template is to choose the
+ the implementation #HfstBasicTransition which is compatible with
+ #HfstBasicTransducer.
+
+ @see HfstBasicTransition
*/
template <class C> class HfstTransition
{
protected:
HfstState target_state; // the state where the transition leads
C transition_data; // the actual transition data
-
- /* Get the number that represents the symbol in the transition data. */
- static unsigned int get_symbol_number
- (const typename C::SymbolType &symbol) {
- return C::get_symbol_number(symbol);
- }
-
+
+ /* Get the number that represents the symbol in the transition data. */
+ static unsigned int get_symbol_number
+ (const typename C::SymbolType &symbol) {
+ return C::get_symbol_number(symbol);
+ }
+
public:
-
+
/** @brief Create a transition leading to state zero with input and
output symbols and weight as given by default constructors
of C::SymbolType and C::WeightType. */
- HfstTransition(): target_state(0)
+ HFSTDLL HfstTransition(): target_state(0)
{}
-
+
/** @brief Create a transition leading to state \a s with input symbol
\a isymbol, output_symbol \a osymbol and weight \a weight. */
- HfstTransition(HfstState s,
- typename C::SymbolType isymbol,
- typename C::SymbolType osymbol,
- typename C::WeightType weight):
+ HFSTDLL HfstTransition(HfstState s,
+ typename C::SymbolType isymbol,
+ typename C::SymbolType osymbol,
+ typename C::WeightType weight):
target_state(s), transition_data(isymbol, osymbol, weight)
{}
-
- HfstTransition(HfstState s,
- unsigned int inumber,
- unsigned int onumber,
- typename C::WeightType weight,
- bool foo):
+
+ HFSTDLL HfstTransition(HfstState s,
+ unsigned int inumber,
+ unsigned int onumber,
+ typename C::WeightType weight,
+ bool foo):
target_state(s), transition_data(inumber, onumber, weight)
{ (void)foo; }
-
+
/** @brief Create a deep copy of transition \a another. */
- HfstTransition(const HfstTransition<C> &another):
+ HFSTDLL HfstTransition(const HfstTransition<C> &another):
target_state(another.target_state),
transition_data(another.transition_data)
- {}
-
+ {}
+
/** @brief Whether this transition is less than transition \a
another. Needed for storing transitions in a set. */
- bool operator<(const HfstTransition<C> &another) const {
+ HFSTDLL bool operator<(const HfstTransition<C> &another) const {
if (target_state == another.target_state)
return (transition_data < another.transition_data);
return (target_state < another.target_state);
- }
-
+ }
+
/** @brief Assign this transition the same value as transition
\a another. */
- void operator=(const HfstTransition<C> &another) {
+ HFSTDLL void operator=(const HfstTransition<C> &another) {
target_state = another.target_state;
transition_data = another.transition_data;
}
-
+
/** @brief Get the target state of the transition. */
- HfstState get_target_state() const {
+ HFSTDLL HfstState get_target_state() const {
return target_state;
}
-
+
/** @brief Get the transition data of the transition. */
- const C & get_transition_data() const {
+ HFSTDLL const C & get_transition_data() const {
return transition_data;
}
-
+
/** @brief Get the input symbol of the transition. */
- typename C::SymbolType get_input_symbol() const {
+ HFSTDLL typename C::SymbolType get_input_symbol() const {
return transition_data.get_input_symbol();
}
/** @brief Get the output symbol of the transition. */
- typename C::SymbolType get_output_symbol() const {
+ HFSTDLL typename C::SymbolType get_output_symbol() const {
return transition_data.get_output_symbol();
}
- /* Get the internal input number of the transition. */
- unsigned int get_input_number() const {
- return transition_data.get_input_number();
- }
+ /* Get the internal input number of the transition. */
+ HFSTDLL unsigned int get_input_number() const {
+ return transition_data.get_input_number();
+ }
- /* Get the internal output number of the transition. */
- unsigned int get_output_number() const {
- return transition_data.get_output_number();
- }
+ /* Get the internal output number of the transition. */
+ HFSTDLL unsigned int get_output_number() const {
+ return transition_data.get_output_number();
+ }
/** @brief Get the weight of the transition. */
- typename C::WeightType get_weight() const {
+ HFSTDLL typename C::WeightType get_weight() const {
return transition_data.get_weight();
}
diff --git a/libhfst/src/implementations/HfstTransitionGraph.h b/libhfst/src/implementations/HfstTransitionGraph.h
index 8641699..da1aba8 100644
--- a/libhfst/src/implementations/HfstTransitionGraph.h
+++ b/libhfst/src/implementations/HfstTransitionGraph.h
@@ -23,18 +23,20 @@
#include "HfstTropicalTransducerTransitionData.h"
#include "HfstFastTransitionData.h"
+ #include "../hfstdll.h"
+
namespace hfst {
class HfstFile {
private:
FILE * file;
public:
- HfstFile();
- ~HfstFile();
- void set_file(FILE * f);
- FILE * get_file();
- void close();
- void write(const char * str);
+ HFSTDLL HfstFile();
+ HFSTDLL ~HfstFile();
+ HFSTDLL void set_file(FILE * f);
+ HFSTDLL FILE * get_file();
+ HFSTDLL void close();
+ HFSTDLL void write(const char * str);
};
@@ -200,13 +202,13 @@
/** @brief Create a graph with one initial state that has state number
zero and is not a final state, i.e. create an empty graph. */
- HfstTransitionGraph(void) {
+ HFSTDLL HfstTransitionGraph(void) {
initialize_alphabet(alphabet);
HfstTransitions tr;
state_vector.push_back(tr);
}
- HfstTransitionGraph(FILE *file) {
+ HFSTDLL HfstTransitionGraph(FILE *file) {
initialize_alphabet(alphabet);
HfstTransitions tr;
state_vector.push_back(tr);
@@ -214,7 +216,7 @@
this->assign(read_in_att_format(file, "@0@", linecount));
}
- HfstTransitionGraph(HfstFile &file) {
+ HFSTDLL HfstTransitionGraph(HfstFile &file) {
initialize_alphabet(alphabet);
HfstTransitions tr;
state_vector.push_back(tr);
@@ -224,7 +226,7 @@
/** @brief The assignment operator. */
- HfstTransitionGraph &operator=(const HfstTransitionGraph &graph)
+ HFSTDLL HfstTransitionGraph &operator=(const HfstTransitionGraph &graph)
{
if (this == &graph)
return *this;
@@ -235,13 +237,13 @@
return *this;
}
- HfstTransitionGraph &assign(const HfstTransitionGraph &graph)
+ HFSTDLL HfstTransitionGraph &assign(const HfstTransitionGraph &graph)
{
return this->operator=(graph);
}
/** @brief Create a deep copy of HfstTransitionGraph \a graph. */
- HfstTransitionGraph(const HfstTransitionGraph &graph) {
+ HFSTDLL HfstTransitionGraph(const HfstTransitionGraph &graph) {
state_vector = graph.state_vector;
final_weight_map = graph.final_weight_map;
alphabet = graph.alphabet;
@@ -250,7 +252,7 @@
/** @brief Create an HfstTransitionGraph equivalent to HfstTransducer
\a transducer. FIXME: move to a separate file */
- HfstTransitionGraph(const hfst::HfstTransducer &transducer) {
+ HFSTDLL HfstTransitionGraph(const hfst::HfstTransducer &transducer) {
HfstTransitionGraph<HfstTropicalTransducerTransitionData>
*fsm = ConversionFunctions::
hfst_transducer_to_hfst_basic_transducer(transducer);
@@ -301,7 +303,7 @@
public:
/* Print the alphabet of the graph to standard error stream. */
- void print_alphabet() const
+ HFSTDLL void print_alphabet() const
{
for (typename HfstTransitionGraphAlphabet::const_iterator it
= alphabet.begin(); it != alphabet.end(); it++)
@@ -348,7 +350,7 @@
@note Usually the user does not have to take care of the alphabet
of a graph. This function can be useful in some special cases. */
- void add_symbol_to_alphabet(const HfstSymbol &symbol) {
+ HFSTDLL void add_symbol_to_alphabet(const HfstSymbol &symbol) {
alphabet.insert(symbol);
}
@@ -356,11 +358,11 @@
@note Use with care, removing symbols that occur in the transitions
of the graph can have unexpected results. */
- void remove_symbol_from_alphabet(const HfstSymbol &symbol) {
+ HFSTDLL void remove_symbol_from_alphabet(const HfstSymbol &symbol) {
alphabet.erase(symbol);
}
- void remove_symbols_from_alphabet(const HfstSymbolSet &symbols) {
+ HFSTDLL void remove_symbols_from_alphabet(const HfstSymbolSet &symbols) {
for (typename HfstSymbolSet::const_iterator it = symbols.begin();
it != symbols.end(); it++)
{
@@ -370,7 +372,7 @@
/** @brief Same as #add_symbol_to_alphabet for each symbol in
\a symbols. */
- void add_symbols_to_alphabet(const HfstSymbolSet &symbols)
+ HFSTDLL void add_symbols_to_alphabet(const HfstSymbolSet &symbols)
{
for (typename HfstSymbolSet::const_iterator it = symbols.begin();
it != symbols.end(); it++)
@@ -379,7 +381,7 @@
}
}
- void add_symbols_to_alphabet(const HfstSymbolPairSet &symbols)
+ HFSTDLL void add_symbols_to_alphabet(const HfstSymbolPairSet &symbols)
{
for (typename HfstSymbolPairSet::const_iterator it = symbols.begin();
it != symbols.end(); it++)
@@ -391,7 +393,7 @@
/* Remove all symbols that are given in \a symbols but do not occur
in transitions of the graph from its alphabet. */
- void prune_alphabet_after_substitution(const std::set<unsigned int> &symbols)
+ HFSTDLL void prune_alphabet_after_substitution(const std::set<unsigned int> &symbols)
{
if (symbols.size() == 0)
return;
@@ -424,7 +426,7 @@
}
- HfstTransitionGraphAlphabet symbols_used()
+ HFSTDLL HfstTransitionGraphAlphabet symbols_used()
{
HfstTransitionGraphAlphabet retval;
for (iterator it = begin(); it != end(); it++)
@@ -450,7 +452,7 @@
Epsilon, unknown and identity \link hfst::String symbols\endlink
are always included in the alphabet. */
- void prune_alphabet(bool force=true) {
+ HFSTDLL void prune_alphabet(bool force=true) {
// Which symbols occur in the graph
HfstTransitionGraphAlphabet symbols_found = symbols_used();
@@ -498,11 +500,11 @@
The HfstSymbols do not necessarily occur in any transitions
of the graph. Epsilon, unknown and identity \link
hfst::String symbols\endlink are always included in the alphabet. */
- const HfstTransitionGraphAlphabet &get_alphabet() const {
+ HFSTDLL const HfstTransitionGraphAlphabet &get_alphabet() const {
return alphabet;
}
- StringPairSet get_transition_pairs() const {
+ HFSTDLL StringPairSet get_transition_pairs() const {
StringPairSet retval;
for (const_iterator it = begin(); it != end(); it++)
@@ -528,7 +530,7 @@
/** @brief Add a new state to this graph and return its number.
@return The next (smallest) free state number. */
- HfstState add_state(void) {
+ HFSTDLL HfstState add_state(void) {
HfstTransitions tr;
state_vector.push_back(tr);
return state_vector.size()-1;
@@ -540,7 +542,7 @@
All states with state number smaller than \a s are also
added to the graph if they did not exist before.
@return \a s*/
- HfstState add_state(HfstState s) {
+ HFSTDLL HfstState add_state(HfstState s) {
while(state_vector.size() <= s) {
HfstTransitions tr;
state_vector.push_back(tr);
@@ -549,14 +551,14 @@
}
/** @brief Get the biggest state number in use. */
- HfstState get_max_state() const {
+ HFSTDLL HfstState get_max_state() const {
return state_vector.size()-1;
}
/** @brief Add a transition \a transition to state \a s.
If state \a s does not exist, it is created. */
- void add_transition(HfstState s, const HfstTransition<C> & transition,
+ HFSTDLL void add_transition(HfstState s, const HfstTransition<C> & transition,
bool add_symbols_to_alphabet=true) {
C data = transition.get_transition_data();
@@ -576,7 +578,7 @@
if they are no longer used in the graph.
If \a state or \a transition does not exist, nothing is done. */
- void remove_transition(HfstState s, const HfstTransition<C> & transition,
+ HFSTDLL void remove_transition(HfstState s, const HfstTransition<C> & transition,
bool remove_symbols_from_alphabet=false)
{
if (! (state_vector.size() > s))
@@ -622,12 +624,12 @@
/** @brief Whether state \a s is final.
FIXME: return positive infinity instead if not final. */
- bool is_final_state(HfstState s) const {
+ HFSTDLL bool is_final_state(HfstState s) const {
return (final_weight_map.find(s) != final_weight_map.end());
}
/** Get the final weight of state \a s in this graph. */
- typename C::WeightType get_final_weight(HfstState s) const {
+ HFSTDLL typename C::WeightType get_final_weight(HfstState s) const {
if (final_weight_map.find(s) != final_weight_map.end())
return final_weight_map.find(s)->second;
HFST_THROW(StateIsNotFinalException);
@@ -637,7 +639,7 @@
to \a weight.
If the state does not exist, it is created. */
- void set_final_weight(HfstState s,
+ HFSTDLL void set_final_weight(HfstState s,
const typename C::WeightType & weight) {
add_state(s);
final_weight_map[s] = weight;
@@ -645,7 +647,7 @@
/** @brief Sort the arcs of this transducer according to input and
output symbols. */
- HfstTransitionGraph &sort_arcs(void)
+ HFSTDLL HfstTransitionGraph &sort_arcs(void)
{
for (typename HfstStates::iterator it = state_vector.begin();
it != state_vector.end();
@@ -662,19 +664,19 @@
the graph.
For an example, see #HfstTransitionGraph */
- iterator begin() { return state_vector.begin(); }
+ HFSTDLL iterator begin() { return state_vector.begin(); }
/** @brief Get a const iterator to the beginning of
states in the graph. */
- const_iterator begin() const { return state_vector.begin(); }
+ HFSTDLL const_iterator begin() const { return state_vector.begin(); }
/** @brief Get an iterator to the end of states (last state + 1)
in the graph. */
- iterator end() { return state_vector.end(); }
+ HFSTDLL iterator end() { return state_vector.end(); }
/** @brief Get a const iterator to the end of states (last state + 1)
in the graph. */
- const_iterator end() const { return state_vector.end(); }
+ HFSTDLL const_iterator end() const { return state_vector.end(); }
/** @brief Get the set of transitions of state \a s in this graph.
@@ -682,7 +684,7 @@
If the state does not exist, a @a StateIndexOutOfBoundsException
is thrown.
*/
- const HfstTransitions & operator[](HfstState s) const
+ HFSTDLL const HfstTransitions & operator[](HfstState s) const
{
if (s >= state_vector.size()) {
HFST_THROW(StateIndexOutOfBoundsException); }
@@ -695,7 +697,7 @@
@see operator[]
*/
- const HfstTransitions & transitions(HfstState s) const
+ HFSTDLL const HfstTransitions & transitions(HfstState s) const
{
return this->operator[](s);
}
@@ -895,7 +897,7 @@
/** @brief Write the graph in xfst text format to ostream \a os.
\a write_weights defines whether weights are printed (todo). */
- void write_in_xfst_format(std::ostream &os, bool write_weights=true)
+ HFSTDLL void write_in_xfst_format(std::ostream &os, bool write_weights=true)
{
(void)write_weights; // todo
unsigned int source_state=0;
@@ -931,7 +933,7 @@
}
// note: unknown and identity are both '?'
- static std::string prologize_symbol(const std::string & symbol)
+ HFSTDLL static std::string prologize_symbol(const std::string & symbol)
{
if (symbol == "0")
return "%0";
@@ -943,14 +945,15 @@
return "?";
if(symbol == "@_IDENTITY_SYMBOL_@")
return "?";
- // prepend a backslash to a double quote
+ // prepend a backslash to a double quote and to a backslash
std::string retval(symbol);
+ replace_all(retval, "\\", "\\\\");
replace_all(retval, "\"", "\\\"");
return retval;
}
// caveat: '?' is always unknown
- static std::string deprologize_symbol(const std::string & symbol)
+ HFSTDLL static std::string deprologize_symbol(const std::string & symbol)
{
if (symbol == "%0")
return "0";
@@ -960,13 +963,15 @@
return "@_EPSILON_SYMBOL_@";
if (symbol == "?")
return "@_UNKNOWN_SYMBOL_@";
- // remove the escaping backslash in front of a double quote
+ // remove the escaping backslash in front of a double quote and
+ // a double quote
std::string retval(symbol);
replace_all(retval, "\\\"", "\"");
+ replace_all(retval, "\\\\", "\\");
return retval;
}
- static void print_prolog_arc_symbols(FILE * file, C data)
+ HFSTDLL static void print_prolog_arc_symbols(FILE * file, C data)
{
std::string symbol = prologize_symbol(data.get_input_symbol());
fprintf(file, "\"%s\"", symbol.c_str());
@@ -980,7 +985,7 @@
}
}
- static void print_prolog_arc_symbols(std::ostream & os, C data)
+ HFSTDLL static void print_prolog_arc_symbols(std::ostream & os, C data)
{
std::string symbol = prologize_symbol(data.get_input_symbol());
os << "\"" << symbol << "\"";
@@ -996,7 +1001,7 @@
/** @brief Write the graph in prolog format to FILE \a file.
\a write_weights defines whether weights are printed (todo). */
- void write_in_prolog_format(FILE * file, const std::string & name,
+ HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
bool write_weights=true)
{
unsigned int source_state=0;
@@ -1017,7 +1022,7 @@
{
if (symbols_used_.find(*it) == symbols_used_.end())
{
- fprintf(file, "symbol(%s, \"%s\").\n", identifier, it->c_str());
+ fprintf(file, "symbol(%s, \"%s\").\n", identifier, prologize_symbol(it->c_str()).c_str());
}
}
@@ -1058,7 +1063,7 @@
/** @brief Write the graph in prolog format to ostream \a os.
\a write_weights defines whether weights are printed (todo). */
- void write_in_prolog_format(std::ostream & os, const std::string & name,
+ HFSTDLL void write_in_prolog_format(std::ostream & os, const std::string & name,
bool write_weights=true)
{
unsigned int source_state=0;
@@ -1079,7 +1084,7 @@
{
if (symbols_used_.find(*it) == symbols_used_.end())
{
- os << "symbol(" << name << ", \"" << *it << "\")" << std::endl;
+ os << "symbol(" << name << ", \"" << prologize_symbol(*it) << "\")" << std::endl;
}
}
@@ -1118,7 +1123,7 @@
// If \a str is of format ".+", change it to .+ and return true.
// Else, return false.
- static bool strip_quotes_from_both_sides(std::string & str)
+ HFSTDLL static bool strip_quotes_from_both_sides(std::string & str)
{
if (str.size() < 3)
return false;
@@ -1131,7 +1136,7 @@
// If \a str is of format .+)\.", change it to .+ and return true.
// Else, return false.
- static bool strip_ending_parenthesis_and_comma(std::string & str)
+ HFSTDLL static bool strip_ending_parenthesis_and_comma(std::string & str)
{
if (str.size() < 3)
return false;
@@ -1141,7 +1146,7 @@
return true;
}
- static bool parse_prolog_network_line(const std::string & line, std::string & name)
+ HFSTDLL static bool parse_prolog_network_line(const std::string & line, std::string & name)
{
// 'network(NAME).'
char namearr[100];
@@ -1160,7 +1165,7 @@
// Get positions of \a c in \a str. If \a esc is precedes
// \a c, \a c is not included.
- static std::vector<unsigned int> get_positions_of_unescaped_char
+ HFSTDLL static std::vector<unsigned int> get_positions_of_unescaped_char
(const std::string & str, char c, char esc)
{
std::vector<unsigned int> retval;
@@ -1183,7 +1188,7 @@
// \a str and store them to \a isymbol and \a osymbol.
// Return whether symbols were successfully extracted.
// \a str must be of format "foo":"bar" or "foo"
- static bool get_prolog_arc_symbols
+ HFSTDLL static bool get_prolog_arc_symbols
(const std::string & str, std::string & isymbol, std::string & osymbol)
{
// find positions of non-escaped double quotes (todo: double double-quote?)
@@ -1243,7 +1248,7 @@
return true;
}
- static bool extract_weight(std::string & symbol, float & weight)
+ HFSTDLL static bool extract_weight(std::string & symbol, float & weight)
{
size_t last_double_quote = symbol.find_last_of('"');
size_t last_space = symbol.find_last_of(' ');
@@ -1272,7 +1277,7 @@
return true;
}
- static bool parse_prolog_arc_line(const std::string & line, HfstTransitionGraph & graph)
+ HFSTDLL static bool parse_prolog_arc_line(const std::string & line, HfstTransitionGraph & graph)
{
// symbolstr can also contain the weight
char namestr[100]; char sourcestr[100];
@@ -1310,7 +1315,7 @@
return true;
}
- static bool parse_prolog_final_line(const std::string & line, HfstTransitionGraph & graph)
+ HFSTDLL static bool parse_prolog_final_line(const std::string & line, HfstTransitionGraph & graph)
{
// 'final(NAME, number).' or 'final(NAME, number, weight).'
char namestr[100];
@@ -1354,7 +1359,7 @@
return true;
}
- static bool parse_prolog_symbol_line(const std::string & line, HfstTransitionGraph & graph)
+ HFSTDLL static bool parse_prolog_symbol_line(const std::string & line, HfstTransitionGraph & graph)
{
// 'symbol(NAME, "foo").'
char namearr[100];
@@ -1376,12 +1381,12 @@
if (! strip_quotes_from_both_sides(symbolstr))
return false;
- graph.add_symbol_to_alphabet(symbolstr);
+ graph.add_symbol_to_alphabet(deprologize_symbol(symbolstr));
return true;
}
// Erase newlines from the end of \a str and return \a str.
- static std::string strip_newlines(std::string & str)
+ HFSTDLL static std::string strip_newlines(std::string & str)
{
for (signed int i=(signed int)str.length()-1; i >= 0; --i)
{
@@ -1397,13 +1402,13 @@
// or from \a file. If successfull, strip the line from newlines,
// increment \a linecount by one and return the line.
// Else, throw an EndOfStreamException.
- static std::string get_stripped_line
+ HFSTDLL static std::string get_stripped_line
(std::istream & is, FILE * file, unsigned int & linecount)
{
char line [255];
if (file == NULL) { /* we use streams */
- if (not is.getline(line,255).eof())
+ if (! is.getline(line,255).eof())
HFST_THROW(EndOfStreamException);
}
else { /* we use FILEs */
@@ -1424,7 +1429,7 @@
read_in_prolog_format(FILE*).
If \a file is NULL, it is ignored and \a is is used.
If \a file is not NULL, it is used and \a is is ignored. */
- static HfstTransitionGraph read_in_prolog_format
+ HFSTDLL static HfstTransitionGraph read_in_prolog_format
(std::istream &is, FILE *file, unsigned int & linecount)
{
@@ -1487,7 +1492,7 @@
HFST_THROW(NotValidPrologFormatException); // this should not happen
}
- static HfstTransitionGraph read_in_prolog_format
+ HFSTDLL static HfstTransitionGraph read_in_prolog_format
(std::istream &is,
unsigned int & linecount)
{
@@ -1496,7 +1501,7 @@
linecount);
}
- static HfstTransitionGraph read_in_prolog_format
+ HFSTDLL static HfstTransitionGraph read_in_prolog_format
(FILE *file,
unsigned int & linecount)
{
@@ -1505,7 +1510,7 @@
file, linecount);
}
- static HfstTransitionGraph read_in_prolog_format
+ HFSTDLL static HfstTransitionGraph read_in_prolog_format
(HfstFile &file,
unsigned int & linecount)
{
@@ -1517,7 +1522,7 @@
/** @brief Write the graph in xfst text format to FILE \a file.
\a write_weights defines whether weights are printed (todo). */
- void write_in_xfst_format(FILE * file, bool write_weights=true)
+ HFSTDLL void write_in_xfst_format(FILE * file, bool write_weights=true)
{
(void)write_weights;
unsigned int source_state=0;
@@ -1558,7 +1563,7 @@
/** @brief Write the graph in AT&T format to ostream \a os.
\a write_weights defines whether weights are printed. */
- void write_in_att_format(std::ostream &os, bool write_weights=true)
+ HFSTDLL void write_in_att_format(std::ostream &os, bool write_weights=true)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
@@ -1605,7 +1610,7 @@
/** @brief Write the graph in AT&T format to FILE \a file.
\a write_weights defines whether weights are printed. */
- void write_in_att_format(FILE *file, bool write_weights=true)
+ HFSTDLL void write_in_att_format(FILE *file, bool write_weights=true)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
@@ -1651,7 +1656,7 @@
}
}
- void write_in_att_format(char * ptr, bool write_weights=true)
+ HFSTDLL void write_in_att_format(char * ptr, bool write_weights=true)
{
unsigned int source_state=0;
size_t cwt = 0; // characters written in total
@@ -1708,7 +1713,7 @@
/** @brief Write the graph in AT&T format to FILE \a file using numbers
instead of symbol names.
\a write_weights defines whether weights are printed. */
- void write_in_att_format_number(FILE *file, bool write_weights=true)
+ HFSTDLL void write_in_att_format_number(FILE *file, bool write_weights=true)
{
unsigned int source_state=0;
for (iterator it = begin(); it != end(); it++)
@@ -1753,7 +1758,7 @@
read_in_att_format(FILE*, std::string).
If \a file is NULL, it is ignored and \a is is used.
If \a file is not NULL, it is used and \a is is ignored. */
- static HfstTransitionGraph read_in_att_format
+ HFSTDLL static HfstTransitionGraph read_in_att_format
(std::istream &is,
FILE *file,
std::string epsilon_symbol,
@@ -1775,7 +1780,7 @@
while(true) {
if (file == NULL) { /* we use streams */
- if (not is.getline(line,255).eof())
+ if (! is.getline(line,255).eof())
break;
}
else { /* we use FILEs */
@@ -1865,7 +1870,7 @@
@pre \a is not at end, otherwise an exception is thrown.
@note Multiple AT&T transducer definitions are separated with
the line "--". */
- static HfstTransitionGraph read_in_att_format
+ HFSTDLL static HfstTransitionGraph read_in_att_format
(std::istream &is,
std::string epsilon_symbol,
unsigned int & linecount)
@@ -1881,7 +1886,7 @@
@pre \a is not at end, otherwise an exception is thrown.
@note Multiple AT&T transducer definitions are separated with
the line "--". */
- static HfstTransitionGraph read_in_att_format
+ HFSTDLL static HfstTransitionGraph read_in_att_format
(FILE *file,
std::string epsilon_symbol,
unsigned int & linecount)
@@ -1891,7 +1896,7 @@
file, epsilon_symbol, linecount);
}
- static HfstTransitionGraph read_in_att_format
+ HFSTDLL static HfstTransitionGraph read_in_att_format
(HfstFile &file,
std::string epsilon_symbol,
unsigned int & linecount)
@@ -2069,7 +2074,7 @@
/* A function that performs in-place removal of all transitions
equivalent to \a sp in the graph. */
- void remove_transitions(const HfstSymbolPair &sp)
+ HFSTDLL void remove_transitions(const HfstSymbolPair &sp)
{
unsigned int in_match = C::get_number(sp.first);
unsigned int out_match = C::get_number(sp.second);
@@ -2244,8 +2249,8 @@
typename HfstSymbolPairSet::const_iterator IT
= substituting_transitions.begin();
- if (not C::is_valid_symbol(IT->first) ||
- not C::is_valid_symbol(IT->second) )
+ if (! C::is_valid_symbol(IT->first) ||
+ ! C::is_valid_symbol(IT->second) )
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransitionGraph::substitute");
@@ -2266,8 +2271,8 @@
while (IT != substituting_transitions.end())
{
- if (not C::is_valid_symbol(IT->first) ||
- not C::is_valid_symbol(IT->second) )
+ if (! C::is_valid_symbol(IT->first) ||
+ ! C::is_valid_symbol(IT->second) )
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransitionGraph::substitute");
@@ -2312,14 +2317,14 @@
/** @brief Substitute \a old_symbol with \a new_symbol in
all transitions. \a input_side and \a output_side define
whether the substitution is made on input and output sides. */
- HfstTransitionGraph &
+ HFSTDLL HfstTransitionGraph &
substitute(const HfstSymbol &old_symbol,
const HfstSymbol &new_symbol,
bool input_side=true,
bool output_side=true) {
- if (not C::is_valid_symbol(old_symbol) ||
- not C::is_valid_symbol(new_symbol) ) {
+ if (! C::is_valid_symbol(old_symbol) ||
+ ! C::is_valid_symbol(new_symbol) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransitionGraph::substitute"); }
@@ -2335,9 +2340,9 @@
// if the substitution is made on both sides.
if (input_side && output_side) {
/* Special symbols are always included in the alphabet */
- if (not is_epsilon(old_symbol) &&
- not is_unknown(old_symbol) &&
- not is_identity(old_symbol)) {
+ if (! is_epsilon(old_symbol) &&
+ ! is_unknown(old_symbol) &&
+ ! is_identity(old_symbol)) {
alphabet.erase(old_symbol); }
}
// Insert the substituting symbol to the alphabet.
@@ -2348,7 +2353,7 @@
return *this;
}
- HfstTransitionGraph &substitute_symbols
+ HFSTDLL HfstTransitionGraph &substitute_symbols
(const HfstSymbolSubstitutions &substitutions)
{ return this->substitute(substitutions); }
@@ -2387,7 +2392,7 @@
return *this;
}
- HfstTransitionGraph &substitute_symbol_pairs
+ HFSTDLL HfstTransitionGraph &substitute_symbol_pairs
(const HfstSymbolPairSubstitutions &substitutions)
{ return this->substitute(substitutions); }
@@ -2397,7 +2402,7 @@
a mapping x:y -> X:Y is found, the transition x:y is replaced
with X:Y. If no mapping is found, the transition remains the same.
*/
- HfstTransitionGraph &substitute
+ HFSTDLL HfstTransitionGraph &substitute
(const HfstSymbolPairSubstitutions &substitutions)
{
// Convert from symbols to numbers
@@ -2422,11 +2427,11 @@
/** @brief Substitute all transitions \a sp with a set of transitions
\a sps. */
- HfstTransitionGraph &substitute
+ HFSTDLL HfstTransitionGraph &substitute
(const HfstSymbolPair &sp, const HfstSymbolPairSet &sps)
{
- if (not C::is_valid_symbol(sp.first) ||
- not C::is_valid_symbol(sp.second) ) {
+ if (! C::is_valid_symbol(sp.first) ||
+ ! C::is_valid_symbol(sp.second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransitionGraph::substitute"); }
@@ -2434,8 +2439,8 @@
for (typename HfstSymbolPairSet::const_iterator it = sps.begin();
it != sps.end(); it++)
{
- if (not C::is_valid_symbol(it->first) ||
- not C::is_valid_symbol(it->second) ) {
+ if (! C::is_valid_symbol(it->first) ||
+ ! C::is_valid_symbol(it->second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransitionGraph::substitute"); }
@@ -2448,14 +2453,14 @@
/** @brief Substitute all transitions \a old_pair with
\a new_pair. */
- HfstTransitionGraph &substitute
+ HFSTDLL HfstTransitionGraph &substitute
(const HfstSymbolPair &old_pair,
const HfstSymbolPair &new_pair)
{
- if (not C::is_valid_symbol(old_pair.first) ||
- not C::is_valid_symbol(new_pair.first) ||
- not C::is_valid_symbol(old_pair.second) ||
- not C::is_valid_symbol(new_pair.second) ) {
+ if (! C::is_valid_symbol(old_pair.first) ||
+ ! C::is_valid_symbol(new_pair.first) ||
+ ! C::is_valid_symbol(old_pair.second) ||
+ ! C::is_valid_symbol(new_pair.second) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
"HfstTransitionGraph::substitute"); }
@@ -2475,7 +2480,7 @@
the original transition \a sp must be replaced. \a func returns
a value indicating whether any substitution must be made, i.e.
whether any transition was inserted into \a sps. */
- HfstTransitionGraph &
+ HFSTDLL HfstTransitionGraph &
substitute(bool (*func)
(const HfstSymbolPair &sp, HfstSymbolPairSet &sps) )
{
@@ -2583,11 +2588,11 @@
\a old_symbol : \a new_symbol (that are substituted)
in this graph.
*/
- HfstTransitionGraph &
+ HFSTDLL HfstTransitionGraph &
substitute(const HfstSymbolPair &sp,
const HfstTransitionGraph &graph) {
- if ( not ( C::is_valid_symbol(sp.first) &&
+ if ( ! ( C::is_valid_symbol(sp.first) &&
C::is_valid_symbol(sp.second) ) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -2675,15 +2680,14 @@
- std::string weight2marker(float weight)
+ HFSTDLL std::string weight2marker(float weight)
{
std::ostringstream o;
o << weight;
return std::string("@") + o.str() + std::string("@");
}
- // HERE
- HfstTransitionGraph & substitute_weights_with_markers() {
+ HFSTDLL HfstTransitionGraph & substitute_weights_with_markers() {
// Go through all current states (we are going to add them)
HfstState limit = state_vector.size();
@@ -2781,7 +2785,7 @@
// ####
typedef std::map<HfstSymbol, HfstTransitionGraph> SubstMap;
- HfstTransitionGraph &
+ HFSTDLL HfstTransitionGraph &
substitute(SubstMap & substitution_map,
bool harmonize) {
@@ -2789,7 +2793,7 @@
for (typename SubstMap::const_iterator it = substitution_map.begin();
it != substitution_map.end(); it++)
{
- if ( not ( C::is_valid_symbol(it->first) ))
+ if ( ! ( C::is_valid_symbol(it->first) ))
{
HFST_THROW_MESSAGE(EmptyStringException,
"HfstTransitionGraph::substitute "
@@ -2907,7 +2911,7 @@
- bool marker2weight(const std::string & str, float & weight)
+ HFSTDLL bool marker2weight(const std::string & str, float & weight)
{
if (str.size() < 3)
return false;
@@ -2923,8 +2927,7 @@
return true;
}
- // HERE
- HfstTransitionGraph & substitute_markers_with_weights() {
+ HFSTDLL HfstTransitionGraph & substitute_markers_with_weights() {
// Go through all states
HfstState limit = state_vector.size();
@@ -3018,10 +3021,10 @@
/** @brief Insert freely any number of \a symbol_pair in
the graph with weight \a weight. */
- HfstTransitionGraph &insert_freely
+ HFSTDLL HfstTransitionGraph &insert_freely
(const HfstSymbolPair &symbol_pair, typename C::WeightType weight)
{
- if ( not ( C::is_valid_symbol(symbol_pair.first) &&
+ if ( ! ( C::is_valid_symbol(symbol_pair.first) &&
C::is_valid_symbol(symbol_pair.second) ) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -3045,7 +3048,7 @@
/** @brief Insert freely any number of any symbol in \a symbol_pairs in
the graph with weight \a weight. */
- HfstTransitionGraph &insert_freely
+ HFSTDLL HfstTransitionGraph &insert_freely
(const HfstSymbolPairSet &symbol_pairs,
typename C::WeightType weight)
{
@@ -3053,7 +3056,7 @@
= symbol_pairs.begin();
symbols_it != symbol_pairs.end(); symbols_it++)
{
- if ( not ( C::is_valid_symbol(symbols_it->first) &&
+ if ( ! ( C::is_valid_symbol(symbols_it->first) &&
C::is_valid_symbol(symbols_it->second) ) ) {
HFST_THROW_MESSAGE
(EmptyStringException,
@@ -3084,7 +3087,7 @@
/** @brief Insert freely any number of \a graph in this
graph. */
- HfstTransitionGraph &insert_freely
+ HFSTDLL HfstTransitionGraph &insert_freely
(const HfstTransitionGraph &graph)
{
HfstSymbol marker_this = C::get_marker(alphabet);
@@ -3132,7 +3135,7 @@
that take two or more graphs as their arguments, unless otherwise
said.
*/
- HfstTransitionGraph &harmonize(HfstTransitionGraph &another)
+ HFSTDLL HfstTransitionGraph &harmonize(HfstTransitionGraph &another)
{
HarmonizeUnknownAndIdentitySymbols foo(*this, another);
return *this;
@@ -3179,7 +3182,7 @@
}
// If not found, create the transition
- if (not transition_found)
+ if (! transition_found)
{
next_state = add_state();
HfstTransition <C> transition(next_state, it->first,
@@ -3213,7 +3216,7 @@
\endverbatim
*/
- HfstTransitionGraph &disjunct
+ HFSTDLL HfstTransitionGraph &disjunct
(const StringPairVector &spv, typename C::WeightType weight)
{
StringPairVector::const_iterator it = spv.begin();
@@ -3230,7 +3233,7 @@
return *this;
}
- bool is_special_symbol(const std::string & symbol)
+ HFSTDLL bool is_special_symbol(const std::string & symbol)
{
if (symbol.size() < 2)
return false;
@@ -3239,7 +3242,7 @@
return false;
}
- HfstTransitionGraph &complete()
+ HFSTDLL HfstTransitionGraph &complete()
{
HfstState failure_state = add_state();
HfstState current_state = 0;
@@ -3277,7 +3280,7 @@
return *this;
}
- StringSet get_flags() const
+ HFSTDLL StringSet get_flags() const
{
StringSet flags;
for (StringSet::const_iterator it = alphabet.begin();
@@ -3293,7 +3296,7 @@
// Whether symbol \a symbol must be purged from transitions and alphabet
// of a transducer after \a flag has been eliminated from the transducer.
// If \a flag is the empty string, all flags have been eliminated.
- bool purge_symbol(const std::string & symbol, const std::string & flag)
+ HFSTDLL bool purge_symbol(const std::string & symbol, const std::string & flag)
{
if (! FdOperation::is_diacritic(symbol))
return false;
@@ -3307,7 +3310,7 @@
// Replace arcs in \a transducer that use flag \a flag with epsilon arcs
// and remove \a flag from alphabet of \a transducer. If \a flag is the empty
// string, replace/remove all flags.
- void flag_purge(const std::string & flag)
+ HFSTDLL void flag_purge(const std::string & flag)
{
// (1) Go through all states and transitions
for (iterator it = begin(); it != end(); it++)
@@ -3347,13 +3350,13 @@
/* Initialize the TopologicalSort by reserving space for a transducer
with biggest state number \a biggest_state_number, */
- void set_biggest_state_number(unsigned int biggest_state_number)
+ HFSTDLL void set_biggest_state_number(unsigned int biggest_state_number)
{
distance_of_state = std::vector<int>(biggest_state_number+1, -1);
}
/* Set the maximum distance of \a state to \a distance, */
- void set_state_at_distance(HfstState state, unsigned int distance,
+ HFSTDLL void set_state_at_distance(HfstState state, unsigned int distance,
bool overwrite)
{
// see that 'state' does not exceed the maximum state number given in initialization
@@ -3382,7 +3385,7 @@
}
/* The states that have a maximum distance of \a distance. */
- const std::set<HfstState> & get_states_at_distance(unsigned int distance)
+ HFSTDLL const std::set<HfstState> & get_states_at_distance(unsigned int distance)
{
// if there is nothing on index 'state',
// push back empty sets of states up to index 'state', including
@@ -3403,7 +3406,7 @@
result contains the set of all states whose (maximum) distance from
the start state is ind.
*/
- std::vector<std::set<HfstState> > topsort(SortDistance dist) const
+ HFSTDLL std::vector<std::set<HfstState> > topsort(SortDistance dist) const
{
typedef std::set<HfstState>::const_iterator StateIt;
unsigned int current_distance = 0; // topological distance
@@ -3454,7 +3457,7 @@
/** The length of longest string accepted by this graph.
If no string is accepted, return -1. */
- int longest_path_size()
+ HFSTDLL int longest_path_size()
{
// get topological maximum distance sort
std::vector<std::set<HfstState> > states_sorted = this->topsort(MaximumDistance);
@@ -3481,7 +3484,7 @@
/** The lengths of strings accepted by this graph, in descending order.
If not string is accepted, return an empty vector. */
- std::vector<unsigned int> path_sizes()
+ HFSTDLL std::vector<unsigned int> path_sizes()
{
std::vector<unsigned int> result;
// get topological maximum distance sort
@@ -3507,7 +3510,7 @@
return result;
}
- bool is_infinitely_ambiguous
+ HFSTDLL bool is_infinitely_ambiguous
(HfstState state,
std::set<HfstState> &epsilon_path_states,
std::vector<unsigned int> &states_handled)
@@ -3547,7 +3550,7 @@
return false;
}
- bool is_infinitely_ambiguous()
+ HFSTDLL bool is_infinitely_ambiguous()
{
std::set<HfstState> epsilon_path_states;
HfstState max_state = this->get_max_state();
@@ -3561,7 +3564,7 @@
return false;
}
- bool is_lookup_infinitely_ambiguous
+ HFSTDLL bool is_lookup_infinitely_ambiguous
(const HfstOneLevelPath& s,
unsigned int& index, HfstState state,
std::set<HfstState> &epsilon_path_states
@@ -3606,7 +3609,7 @@
/* CASE 2: Other input symbols consume a symbol in the lookup path s,
so they can be added only if the end of the lookup path s has not
been reached. */
- else if (not only_epsilons)
+ else if (! only_epsilons)
{
bool continu = false;
if (it->get_input_symbol().compare(s.second.at(index)) == 0)
@@ -3635,7 +3638,7 @@
return false;
}
- bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s)
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const HfstOneLevelPath & s)
{
std::set<HfstState> epsilon_path_states;
epsilon_path_states.insert(0);
@@ -3645,12 +3648,12 @@
epsilon_path_states);
}
- bool is_lookup_infinitely_ambiguous(const StringVector & s)
+ HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s)
{
std::set<HfstState> epsilon_path_states;
epsilon_path_states.insert(0);
unsigned int index=0;
- HfstOneLevelPath path(0, s);
+ HfstOneLevelPath path((float)0, s);
return is_lookup_infinitely_ambiguous(path, index, INITIAL_STATE,
epsilon_path_states);
@@ -3658,7 +3661,7 @@
- static void push_back_to_two_level_path
+ HFSTDLL static void push_back_to_two_level_path
(HfstTwoLevelPath &path,
const StringPair &sp,
const float &weight)
@@ -3667,7 +3670,7 @@
path.first = path.first + weight;
}
- static void pop_back_from_two_level_path
+ HFSTDLL static void pop_back_from_two_level_path
(HfstTwoLevelPath &path,
const float &weight)
{
@@ -3675,7 +3678,7 @@
path.first = path.first - weight;
}
- static void add_to_results
+ HFSTDLL static void add_to_results
(HfstTwoLevelPaths &results,
HfstTwoLevelPath &path_so_far,
const float &final_weight,
@@ -3698,7 +3701,7 @@
path_so_far.first = path_so_far.first - final_weight;
}
- static bool is_possible_transition
+ HFSTDLL static bool is_possible_transition
(const HfstBasicTransition &transition,
const StringVector &lookup_path,
const unsigned int &lookup_index,
@@ -3708,7 +3711,7 @@
std::string isymbol = transition.get_input_symbol();
// If we are not at the end of lookup_path,
- if (not (lookup_index == (unsigned int)lookup_path.size()))
+ if (! (lookup_index == (unsigned int)lookup_path.size()))
{
// we can go further if the current symbol in lookup_path
// matches to the input symbol of the transition, i.e
@@ -3740,7 +3743,7 @@
return false;
}
- void lookup_fd
+ HFSTDLL void lookup_fd
(const StringVector &lookup_path,
HfstTwoLevelPaths &results,
HfstState state,
@@ -3752,7 +3755,7 @@
float * max_weight = NULL)
{
// Check whether the number of input epsilon cycles is exceeded
- if (not Eh.can_continue(state)) {
+ if (! Eh.can_continue(state)) {
return;
}
// Check whether the maximum weight is exceeded
@@ -3844,7 +3847,7 @@
}
- void lookup_fd
+ HFSTDLL void lookup_fd
(const StringVector &lookup_path,
HfstTwoLevelPaths &results,
size_t infinite_cutoff,
@@ -3860,7 +3863,7 @@
}
- void check_regexp_state_for_cycle(HfstState s, const std::set<HfstState> & states_visited)
+ HFSTDLL void check_regexp_state_for_cycle(HfstState s, const std::set<HfstState> & states_visited)
{
if (states_visited.find(s) != states_visited.end())
{
@@ -3869,7 +3872,7 @@
}
// Returns whether tr is "^]":"^]". If tr is not allowed, throws an error message.
- bool check_regexp_transition_end(const HfstBasicTransition & tr, bool input_side)
+ HFSTDLL bool check_regexp_transition_end(const HfstBasicTransition & tr, bool input_side)
{
std::string istr = tr.get_input_symbol();
std::string ostr = tr.get_output_symbol();
@@ -3900,7 +3903,7 @@
// [x:y]* "^]" (x and y cannot be "^]" or "^[") starting from state \a s. The resulting
// paths are stored in \a full_paths. \a path is used to keep track of each path so
// far. Weights are currently ignored.
- void find_regexp_paths
+ HFSTDLL void find_regexp_paths
(HfstState s,
std::set<HfstState> & states_visited,
std::vector<std::pair<std::string, std::string> > & path,
@@ -3952,7 +3955,7 @@
// or loops are encountered on a regexp path. Final states are allowed on regexp paths as they are also
// allowed by Xerox tools.
// Weights are currently ignored.
- void find_regexp_paths
+ HFSTDLL void find_regexp_paths
(HfstState s,
std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths,
bool input_side)
@@ -3985,7 +3988,7 @@
// Find all subpaths of form "^[" [x:y]* "^]" (x and y cannot be "^[" or "^]") and return them.
// retval[start_state] == vector(pair(end_state, vector(pair(isymbol,osymbol) ) ) )
// Weights are currently ignored.
- HfstReplacementsMap find_replacements(bool input_side)
+ HFSTDLL HfstReplacementsMap find_replacements(bool input_side)
{
HfstReplacementsMap replacements;
unsigned int state = 0;
@@ -4009,7 +4012,7 @@
// \a graph to state \a state2 with a weight of that state's final weight. Final states of
// \a graph as well as its initial state are made normal non-final states when copying \a graph.
// Todo: copy alphabet? harmonize graphs?
- void insert_transducer(HfstState state1, HfstState state2, const HfstTransitionGraph & graph)
+ HFSTDLL void insert_transducer(HfstState state1, HfstState state2, const HfstTransitionGraph & graph)
{
HfstState offset = add_state();
HfstState source_state=0;
@@ -4076,7 +4079,7 @@
// A function used by find_matches.
// Copy matching transition tr1/tr2 to state \a state in \a intersection and return
// the target state of that transition. Also make that state final, if needed.
- static HfstState handle_match(const HfstTransitionGraph & graph1, const HfstTransition <C> & tr1,
+ HFSTDLL static HfstState handle_match(const HfstTransitionGraph & graph1, const HfstTransition <C> & tr1,
const HfstTransitionGraph & graph2, const HfstTransition <C> & tr2,
HfstTransitionGraph & intersection, HfstState state, StateMap & state_map)
@@ -4115,7 +4118,7 @@
//
// @pre \a graph1 and \a graph2 must be arc-sorted (via sort_arcs()) to make transition matching faster.
// @pre \a graph1 and \a graph2 must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
- static void find_matches
+ HFSTDLL static void find_matches
(HfstTransitionGraph & graph1, HfstState state1, HfstTransitionGraph & graph2, HfstState state2,
HfstTransitionGraph & intersection, HfstState state, StateMap & state_map, std::set<HfstState> & agenda)
{
@@ -4178,7 +4181,7 @@
return;
}
- static HfstTransitionGraph intersect
+ HFSTDLL static HfstTransitionGraph intersect
(HfstTransitionGraph & graph1, HfstTransitionGraph & graph2)
{
HfstTransitionGraph retval;
@@ -4199,15 +4202,10 @@
return retval;
}
-
-
-
- // HERE BEGINS
-
// A function used by find_matches_for_merge
// Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
// the target state of that transition. Also make that state final, if needed.
- static HfstState handle_non_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
+ HFSTDLL static HfstState handle_non_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
const HfstTransitionGraph & merger, HfstState merger_target,
HfstTransitionGraph & result, HfstState result_state, StateMap & state_map)
@@ -4233,7 +4231,7 @@
// A function used by find_matches_for_merge
// Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
// the target state of that transition. Also make that state final, if needed.
- static HfstState handle_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
+ HFSTDLL static HfstState handle_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
const HfstTransitionGraph & merger, const HfstTransition <C> & merger_transition,
HfstTransitionGraph & result, HfstState result_state, StateMap & state_map, std::set<std::string> & markers_added)
{
@@ -4267,7 +4265,7 @@
- static bool is_list_symbol(const C & transition_data, const std::map<std::string, std::set<std::string> > & list_symbols)
+ HFSTDLL static bool is_list_symbol(const C & transition_data, const std::map<std::string, std::set<std::string> > & list_symbols)
{
std::string isymbol = transition_data.get_input_symbol();
std::string osymbol = transition_data.get_output_symbol();
@@ -4327,7 +4325,7 @@
//
// @pre \a graph and \a merger must be arc-sorted (via sort_arcs()) to make transition matching faster.
// @pre \a graph and \a merger must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
- static void find_matches_for_merge
+ HFSTDLL static void find_matches_for_merge
(HfstTransitionGraph & graph, HfstState graph_state, HfstTransitionGraph & merger, HfstState merger_state,
HfstTransitionGraph & result, HfstState result_state, StateMap & state_map, std::set<HfstState> & agenda,
const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added)
@@ -4393,7 +4391,7 @@
return;
}
- static HfstTransitionGraph merge
+ HFSTDLL static HfstTransitionGraph merge
(HfstTransitionGraph & graph, HfstTransitionGraph & merger, const std::map<std::string, std::set<std::string> > & list_symbols, std::set<std::string> & markers_added)
{
HfstTransitionGraph result;
diff --git a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
index 2c96f67..408a459 100644
--- a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
+++ b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
@@ -7,6 +7,8 @@
#include <vector>
#include "../HfstExceptionDefs.h"
+#include "../hfstdll.h"
+
namespace hfst {
namespace implementations {
@@ -45,17 +47,17 @@ namespace hfst {
typedef std::map<SymbolType, unsigned int, string_comparison>
Symbol2NumberMap;
- static SymbolType get_epsilon()
+ HFSTDLL static SymbolType get_epsilon()
{
return SymbolType("@_EPSILON_SYMBOL_@");
}
- static SymbolType get_unknown()
+ HFSTDLL static SymbolType get_unknown()
{
return SymbolType("@_UNKNOWN_SYMBOL_@");
}
- static SymbolType get_identity()
+ HFSTDLL static SymbolType get_identity()
{
return SymbolType("@_IDENTITY_SYMBOL_@");
}
@@ -63,13 +65,13 @@ namespace hfst {
public: /* FIXME: Should be private. */
/* Maps that contain information of the mappings between strings
and numbers */
- static Number2SymbolVector number2symbol_map;
- static Symbol2NumberMap symbol2number_map;
+ HFSTDLL static Number2SymbolVector number2symbol_map;
+ HFSTDLL static Symbol2NumberMap symbol2number_map;
/* The biggest number in use. */
- static unsigned int max_number;
+ HFSTDLL static unsigned int max_number;
/* Get the biggest number used to represent a symbol. */
- static unsigned int get_max_number() {
+ HFSTDLL static unsigned int get_max_number() {
return max_number;
}
@@ -172,7 +174,7 @@ namespace hfst {
WeightType weight;
public:
- void print_transition_data()
+ HFSTDLL void print_transition_data()
{
fprintf(stderr, "%i:%i %f\n",
input_number, output_number, weight);
@@ -182,12 +184,12 @@ namespace hfst {
/** @brief Create a HfstTropicalTransducerTransitionData with
epsilon input and output strings and weight zero. */
- HfstTropicalTransducerTransitionData():
+ HFSTDLL HfstTropicalTransducerTransitionData():
input_number(0), output_number(0), weight(0) {}
/** @brief Create a deep copy of HfstTropicalTransducerTransitionData
\a data. */
- HfstTropicalTransducerTransitionData
+ HFSTDLL HfstTropicalTransducerTransitionData
(const HfstTropicalTransducerTransitionData &data) {
input_number = data.input_number;
output_number = data.output_number;
@@ -197,7 +199,7 @@ namespace hfst {
/** @brief Create a HfstTropicalTransducerTransitionData with
input symbol \a isymbol, output symbol \a osymbol
and weight \a weight. */
- HfstTropicalTransducerTransitionData(SymbolType isymbol,
+ HFSTDLL HfstTropicalTransducerTransitionData(SymbolType isymbol,
SymbolType osymbol,
WeightType weight) {
if (isymbol == "" || osymbol == "")
@@ -211,7 +213,7 @@ namespace hfst {
this->weight = weight;
}
- HfstTropicalTransducerTransitionData
+ HFSTDLL HfstTropicalTransducerTransitionData
(unsigned int inumber,
unsigned int onumber,
WeightType weight) {
@@ -221,45 +223,45 @@ namespace hfst {
}
/** @brief Get the input symbol. */
- const SymbolType &get_input_symbol() const {
+ HFSTDLL const SymbolType &get_input_symbol() const {
return get_symbol(input_number);
}
/** @brief Get the output symbol. */
- const SymbolType &get_output_symbol() const {
+ HFSTDLL const SymbolType &get_output_symbol() const {
return get_symbol(output_number);
}
- unsigned int get_input_number() const {
+ HFSTDLL unsigned int get_input_number() const {
return input_number;
}
- unsigned int get_output_number() const {
+ HFSTDLL unsigned int get_output_number() const {
return output_number;
}
/** @brief Get the weight. */
- WeightType get_weight() const {
+ HFSTDLL WeightType get_weight() const {
return weight;
}
/* Are these needed? */
- static bool is_epsilon(const SymbolType &symbol) {
+ HFSTDLL static bool is_epsilon(const SymbolType &symbol) {
return (symbol.compare("@_EPSILON_SYMBOL_@") == 0);
}
- static bool is_unknown(const SymbolType &symbol) {
+ HFSTDLL static bool is_unknown(const SymbolType &symbol) {
return (symbol.compare("@_UNKNOWN_SYMBOL_@") == 0);
}
- static bool is_identity(const SymbolType &symbol) {
+ HFSTDLL static bool is_identity(const SymbolType &symbol) {
return (symbol.compare("@_IDENTITY_SYMBOL_@") == 0);
}
- static bool is_valid_symbol(const SymbolType &symbol) {
+ HFSTDLL static bool is_valid_symbol(const SymbolType &symbol) {
if (symbol == "")
return false;
return true;
}
- static SymbolType get_marker(const SymbolTypeSet &sts) {
+ HFSTDLL static SymbolType get_marker(const SymbolTypeSet &sts) {
(void)sts;
return SymbolType("@_MARKER_SYMBOL_@");
}
@@ -269,7 +271,7 @@ namespace hfst {
/internal is it too slow if string comparison is used instead?
*/
- bool operator<(const HfstTropicalTransducerTransitionData &another)
+ HFSTDLL bool operator<(const HfstTropicalTransducerTransitionData &another)
const {
if (input_number < another.input_number )
return true;
@@ -283,7 +285,7 @@ namespace hfst {
}
// same as operator< but weight is ignored
- bool less_than_ignore_weight(const HfstTropicalTransducerTransitionData &another)
+ HFSTDLL bool less_than_ignore_weight(const HfstTropicalTransducerTransitionData &another)
const {
if (input_number < another.input_number )
return true;
@@ -296,7 +298,7 @@ namespace hfst {
return false;
}
- void operator=(const HfstTropicalTransducerTransitionData &another)
+ HFSTDLL void operator=(const HfstTropicalTransducerTransitionData &another)
{
input_number = another.input_number;
output_number = another.output_number;
@@ -318,7 +320,7 @@ namespace hfst {
// HfstTropicalTransducerTransitionData..
class Number2SymbolVectorInitializer {
public:
- Number2SymbolVectorInitializer
+ HFSTDLL Number2SymbolVectorInitializer
(HfstTropicalTransducerTransitionData::Number2SymbolVector &vect) {
vect.push_back(std::string("@_EPSILON_SYMBOL_@"));
vect.push_back(std::string("@_UNKNOWN_SYMBOL_@"));
@@ -328,7 +330,7 @@ namespace hfst {
class Symbol2NumberMapInitializer {
public:
- Symbol2NumberMapInitializer
+ HFSTDLL Symbol2NumberMapInitializer
(HfstTropicalTransducerTransitionData::Symbol2NumberMap &map) {
map["@_EPSILON_SYMBOL_@"] = 0;
map["@_UNKNOWN_SYMBOL_@"] = 1;
diff --git a/libhfst/src/implementations/LogWeightTransducer.cc b/libhfst/src/implementations/LogWeightTransducer.cc
index 28ebef5..fce1f52 100644
--- a/libhfst/src/implementations/LogWeightTransducer.cc
+++ b/libhfst/src/implementations/LogWeightTransducer.cc
@@ -72,7 +72,7 @@ namespace hfst { namespace implementations
fst::SymbolTable st(t->InputSymbols()->Name());
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
if (it.Symbol() != symbol) {
st.AddSymbol(it.Symbol(), it.Value());
}
@@ -86,7 +86,7 @@ namespace hfst { namespace implementations
StringSet s;
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
s.insert( std::string(it.Symbol()) );
}
return s;
@@ -113,7 +113,7 @@ namespace hfst { namespace implementations
// find the number-to-number mappings for transducer t1
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t1->InputSymbols()));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
km [ (unsigned int)it.Value() ]
= (unsigned int) t2->InputSymbols()->Find( it.Symbol() );
}
@@ -125,7 +125,7 @@ namespace hfst { namespace implementations
(LogFst *t, NumberNumberMap &km)
{
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::MutableArcIterator<LogFst> aiter(t,s);
@@ -146,7 +146,7 @@ namespace hfst { namespace implementations
LogFst * LogWeightTransducer::set_final_weights(LogFst * t, float weight)
{
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != LogWeight::Zero() )
@@ -171,7 +171,7 @@ namespace hfst { namespace implementations
(LogFst * t,float (*func)(float f))
{
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != LogWeight::Zero() )
@@ -208,7 +208,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -241,7 +241,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -289,7 +289,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -321,7 +321,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -371,7 +371,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -406,7 +406,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -456,7 +456,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -491,7 +491,7 @@ namespace hfst { namespace implementations
}
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -634,12 +634,12 @@ namespace hfst { namespace implementations
LogFst * result = new LogFst();
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
result->AddState();
// go through all states in this
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
// create new state in result, if needed
StateId s = siter.Value();
@@ -745,7 +745,7 @@ namespace hfst { namespace implementations
{
unsigned int retval=0;
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
retval++;
return retval;
}
@@ -792,14 +792,14 @@ namespace hfst { namespace implementations
LogFst *harmonized_t1;
LogFst *harmonized_t2;
- if (not unknown_symbols_in_use)
+ if (! unknown_symbols_in_use)
harmonized_t1 = t1;
else {
harmonized_t1 = expand_arcs(t1, unknown_t1, unknown_symbols_in_use);
harmonized_t1->SetInputSymbols(t1->InputSymbols());
}
- if (not unknown_symbols_in_use)
+ if (! unknown_symbols_in_use)
harmonized_t2 = t2;
else {
harmonized_t2 = expand_arcs(t2, unknown_t2, unknown_symbols_in_use);
@@ -1115,7 +1115,7 @@ namespace hfst { namespace implementations
bool LogWeightTransducer::is_automaton(LogFst * t)
{
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::ArcIterator<LogFst> aiter(*t,s);
@@ -1170,8 +1170,8 @@ namespace hfst { namespace implementations
t->SetStart(s1);
StateId s2=s1; // final state
- if (not sps.empty()) {
- if (not cyclic)
+ if (! sps.empty()) {
+ if (! cyclic)
s2 = t->AddState();
for (StringPairSet::const_iterator it = sps.begin();
it != sps.end();
@@ -1240,8 +1240,8 @@ namespace hfst { namespace implementations
t->SetStart(s1);
StateId s2=s1; // final state
- if (not nps.empty()) {
- if (not cyclic)
+ if (! nps.empty()) {
+ if (! cyclic)
s2 = t->AddState();
for (NumberPairSet::const_iterator it = nps.begin();
it != nps.end();
@@ -1312,7 +1312,7 @@ namespace hfst { namespace implementations
void print_att_number(LogFst *t, FILE * ofile) {
fprintf(ofile, "initial state: %i\n", t->Start());
for (fst::StateIterator<LogFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != LogWeight::Zero() )
@@ -1634,7 +1634,7 @@ namespace hfst { namespace implementations
fst::SymbolTable * st = tc->InputSymbols()->Copy();
assert(st != NULL);
for (fst::StateIterator<LogFst> siter(*tc);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::MutableArcIterator<LogFst> aiter(tc,s);
@@ -1691,12 +1691,13 @@ namespace hfst { namespace implementations
{
assert(t->InputSymbols() != NULL);
SymbolTable * st = t->InputSymbols()->Copy();
+ // TODO: cl.exe: conversion from 'int64' to 'unsigned int'
pair<unsigned int, unsigned int> old_pair
- (st->AddSymbol(old_symbol_pair.first),
- st->AddSymbol(old_symbol_pair.second));
+ ((unsigned int)st->AddSymbol(old_symbol_pair.first),
+ (unsigned int)st->AddSymbol(old_symbol_pair.second));
pair<unsigned int, unsigned int> new_pair
- (st->AddSymbol(new_symbol_pair.first),
- st->AddSymbol(new_symbol_pair.second));
+ ((unsigned int)st->AddSymbol(new_symbol_pair.first),
+ (unsigned int)st->AddSymbol(new_symbol_pair.second));
LogFst * retval = substitute(t, old_pair, new_pair);
retval->SetInputSymbols(st);
delete st;
@@ -1715,7 +1716,7 @@ namespace hfst { namespace implementations
for( int i = 0; i < states; ++i ) {
for (fst::MutableArcIterator<LogFst> it(t,i);
- not it.Done();
+ ! it.Done();
it.Next()) {
fst::LogArc arc = it.Value();
@@ -1799,7 +1800,7 @@ namespace hfst { namespace implementations
for( int i = 0; i < states; ++i ) {
for (fst::MutableArcIterator<LogFst> it(t,i);
- not it.Done();
+ ! it.Done();
it.Next()) {
fst::LogArc arc = it.Value();
@@ -1938,7 +1939,7 @@ namespace hfst { namespace implementations
}
}
- if (not transition_found) {
+ if (! transition_found) {
StateId new_state = t->AddState();
t->AddArc(s, LogArc(inumber,onumber,0,new_state));
s = new_state;
@@ -1973,7 +1974,7 @@ namespace hfst { namespace implementations
}
}
- if (not transition_found) {
+ if (! transition_found) {
StateId new_state = t->AddState();
t->AddArc(s, LogArc(inumber,onumber,0,new_state));
s = new_state;
@@ -2063,7 +2064,7 @@ namespace hfst { namespace implementations
{
LogFst * t_copy = new LogFst(*t);
for (fst::StateIterator<LogFst> iter(*t);
- not iter.Done(); iter.Next())
+ ! iter.Done(); iter.Next())
{
if (t_copy->Final(iter.Value()) != fst::LogWeight::Zero())
{ t_copy->SetFinal(iter.Value(),f); }
@@ -2083,7 +2084,7 @@ namespace hfst { namespace implementations
!aiter.Done();
aiter.Next())
{
- if ((aiter.Value().ilabel == ilabel) and
+ if ((aiter.Value().ilabel == ilabel) &&
(aiter.Value().olabel == olabel))
{ return aiter.Position(); }
}
diff --git a/libhfst/src/implementations/LogWeightTransducer.h b/libhfst/src/implementations/LogWeightTransducer.h
index 8d9de51..1bfdc96 100644
--- a/libhfst/src/implementations/LogWeightTransducer.h
+++ b/libhfst/src/implementations/LogWeightTransducer.h
@@ -16,12 +16,15 @@
#include "HfstExceptionDefs.h"
#include "HfstFlagDiacritics.h"
-#ifdef WINDOWS
-#include "back-ends/openfstwin/src/include/fst/fstlib.h"
+#if HAVE_CONFIG_H
+ #include "../../../config.h"
+#endif
+
+#ifdef _MSC_VER
+ #include "back-ends/openfstwin/src/include/fst/fstlib.h"
#else
-#include "../../../config.h"
-#include "back-ends/openfst/src/include/fst/fstlib.h"
-#endif // WINDOWS
+ #include "back-ends/openfst/src/include/fst/fstlib.h"
+#endif // _MSC_VER
#include "HfstExtractStrings.h"
#include <cstdio>
diff --git a/libhfst/src/implementations/Makefile.am b/libhfst/src/implementations/Makefile.am
index ada3dee..56f6b69 100644
--- a/libhfst/src/implementations/Makefile.am
+++ b/libhfst/src/implementations/Makefile.am
@@ -24,7 +24,7 @@ IMPLEMENTATION_SRCS=HfstTransitionGraph.cc \
HfstTropicalTransducerTransitionData.cc \
ConvertSfstTransducer.cc ConvertTropicalWeightTransducer.cc \
ConvertLogWeightTransducer.cc ConvertFomaTransducer.cc \
- ConvertOlTransducer.cc \
+ ConvertOlTransducer.cc ConvertXfsmTransducer.cc \
compose_intersect/ComposeIntersectRulePair.cc \
compose_intersect/ComposeIntersectLexicon.cc \
compose_intersect/ComposeIntersectRule.cc \
@@ -42,6 +42,10 @@ else
AM_CPPFLAGS += -I${top_srcdir}/back-ends/openfst/src/include
endif
+if GENERATE_LEXC_WRAPPER
+ AM_CPPFLAGS += -DZLIB
+endif
+
# the bridges
if WANT_SFST
MAYBE_SFST=SfstTransducer.cc
@@ -55,6 +59,9 @@ endif
if WANT_FOMA
MAYBE_FOMA=FomaTransducer.cc
endif
+if WANT_XFSM
+MAYBE_XFSM=XfsmTransducer.cc
+endif
# Add here your transducer library
#if WANT_MY_TRANSDUCER_LIBRARY
#MAYBE_MY_TRANSDUCER_LIBRARY=MyTransducerLibraryTransducer.cc
@@ -63,7 +70,7 @@ if WANT_HFSTOL
MAYBE_HFSTOL=HfstOlTransducer.cc
endif
-BRIDGE_SRCS=$(MAYBE_SFST) $(MAYBE_OPENFST) $(MAYBE_FOMA) $(MAYBE_HFSTOL) # $(MAYBE_MY_TRANSDUCER_LIBRARY)
+BRIDGE_SRCS=$(MAYBE_SFST) $(MAYBE_OPENFST) $(MAYBE_FOMA) $(MAYBE_HFSTOL) $(MAYBE_XFSM) # $(MAYBE_MY_TRANSDUCER_LIBRARY)
if WANT_HFSTOL
HFST_OL_SRCS=\
@@ -86,6 +93,7 @@ implinclude_HEADERS = \
FomaTransducer.h \
LogWeightTransducer.h \
SfstTransducer.h TropicalWeightTransducer.h \
+ XfsmTransducer.h \
HfstOlTransducer.h HfstTransitionGraph.h HfstTransition.h \
HfstTropicalTransducerTransitionData.h \
HfstFastTransitionData.h \
@@ -130,11 +138,14 @@ endif
if WANT_SFST
SFST_TSTS=SfstTransducer
endif
+if WANT_XFSM
+XFSM_TSTS=XfsmTransducer
+endif
LIBHFST_TSTS=HfstTransitionGraph ConvertTransducerFormat \
ConvertSfstTransducer ConvertTropicalWeightTransducer \
ConvertLogWeightTransducer ConvertFomaTransducer \
- ConvertOlTransducer \
+ ConvertXfsmTransducer ConvertOlTransducer \
compose_intersect/ComposeIntersectRulePair \
compose_intersect/ComposeIntersectLexicon \
compose_intersect/ComposeIntersectRule \
@@ -142,11 +153,14 @@ LIBHFST_TSTS=HfstTransitionGraph ConvertTransducerFormat \
compose_intersect/ComposeIntersectUtilities
# HfstConstantTransducer
-check_PROGRAMS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS)
+check_PROGRAMS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS) $(XFSM_TSTS)
FomaTransducer_SOURCES=FomaTransducer.cc
FomaTransducer_CXXFLAGS=-DMAIN_TEST
FomaTransducer_LDADD=../libhfst.la
+XfsmTransducer_SOURCES=XfsmTransducer.cc
+XfsmTransducer_CXXFLAGS=-DMAIN_TEST
+XfsmTransducer_LDADD=../libhfst.la
HfstOlTransducer_SOURCES=HfstOlTransducer.cc
HfstOlTransducer_CXXFLAGS=-DMAIN_TEST
HfstOlTransducer_LDADD=../libhfst.la
@@ -177,6 +191,9 @@ ConvertLogWeightTransducer_LDADD=../libhfst.la
ConvertFomaTransducer_SOURCES=ConvertFomaTransducer.cc
ConvertFomaTransducer_CXXFLAGS=-DMAIN_TEST
ConvertFomaTransducer_LDADD=../libhfst.la
+ConvertXfsmTransducer_SOURCES=ConvertXfsmTransducer.cc
+ConvertXfsmTransducer_CXXFLAGS=-DMAIN_TEST
+ConvertXfsmTransducer_LDADD=../libhfst.la
ConvertOlTransducer_SOURCES=ConvertOlTransducer.cc
ConvertOlTransducer_CXXFLAGS=-DMAIN_TEST
ConvertOlTransducer_LDADD=../libhfst.la
@@ -206,4 +223,4 @@ compose_intersect_ComposeIntersectUtilities_LDADD=../libhfst.la
#HfstConstantTransducer_CXXFLAGS=-DMAIN_TEST
#HfstConstantTransducer_LDADD=../libhfst.la
-TESTS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS)
\ No newline at end of file
+TESTS=$(FOMA_TSTS) $(HFSTOL_TSTS) $(OFST_TSTS) $(SFST_TSTS) $(LIBHFST_TSTS) $(XFSM_TSTS)
\ No newline at end of file
diff --git a/libhfst/src/implementations/TropicalWeightTransducer.cc b/libhfst/src/implementations/TropicalWeightTransducer.cc
index b72cb8e..7ce89e3 100644
--- a/libhfst/src/implementations/TropicalWeightTransducer.cc
+++ b/libhfst/src/implementations/TropicalWeightTransducer.cc
@@ -111,7 +111,7 @@ namespace hfst {
fst::SymbolTable st(t->InputSymbols()->Name());
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
- not it.Done(); it.Next() ) {
+ !it.Done(); it.Next() ) {
if (it.Symbol() != symbol) {
st.AddSymbol(it.Symbol(), it.Value());
}
@@ -158,7 +158,7 @@ namespace hfst {
StringSet s;
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
s.insert( std::string(it.Symbol()) );
}
return s;
@@ -181,7 +181,7 @@ namespace hfst {
unsigned int biggest_number=0;
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t->InputSymbols()));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
if (it.Value() > biggest_number)
biggest_number = it.Value();
}
@@ -216,12 +216,12 @@ namespace hfst {
// find the number-to-number mappings for transducer t1
for ( fst::SymbolTableIterator it
= fst::SymbolTableIterator(*(t1->InputSymbols()));
- not it.Done(); it.Next() ) {
+ ! it.Done(); it.Next() ) {
km [ (unsigned int)it.Value() ]
= (unsigned int) t2->InputSymbols()->Find( it.Symbol() );
assert(it.Value() >= 0);
- assert(not (t2->InputSymbols()->Find( it.Symbol() ) < 0));
+ assert(! (t2->InputSymbols()->Find( it.Symbol() ) < 0));
}
return km;
@@ -232,7 +232,7 @@ namespace hfst {
(StdVectorFst *t, NumberNumberMap &km)
{
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::MutableArcIterator<StdVectorFst> aiter(t,s);
@@ -258,7 +258,7 @@ namespace hfst {
(StdVectorFst * t, float weight, bool increment)
{
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != TropicalWeight::Zero() )
@@ -295,7 +295,7 @@ namespace hfst {
(StdVectorFst * t,float (*func)(float f))
{
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != TropicalWeight::Zero() )
@@ -333,7 +333,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -367,7 +367,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -416,7 +416,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -449,7 +449,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -500,7 +500,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -536,7 +536,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -586,7 +586,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s == initial_state) {
@@ -622,7 +622,7 @@ namespace hfst {
}
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if (s != initial_state) {
@@ -766,12 +766,12 @@ namespace hfst {
StdVectorFst * result = new StdVectorFst();
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
result->AddState();
// go through all states in this
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
// create new state in result, if needed
StateId s = siter.Value();
@@ -808,13 +808,13 @@ namespace hfst {
for (StringSet::iterator it1 = unknown.begin();
it1 != unknown.end(); it1++)
{
- if (not FdOperation::is_diacritic(*it1)) {
+ if (! FdOperation::is_diacritic(*it1)) {
int64 inumber = is->Find(*it1);
for (StringSet::iterator it2 = unknown.begin();
it2 != unknown.end(); it2++)
{
- if (not FdOperation::is_diacritic(*it2)) {
+ if (! FdOperation::is_diacritic(*it2)) {
int64 onumber = is->Find(*it2);
if (inumber != onumber)
result->AddArc(result_s,
@@ -838,7 +838,7 @@ namespace hfst {
for (StringSet::iterator it = unknown.begin();
it != unknown.end(); it++)
{
- if (not FdOperation::is_diacritic(*it)) {
+ if (! FdOperation::is_diacritic(*it)) {
int64 number = is->Find(*it);
result->AddArc(result_s,
StdArc(number, number,
@@ -851,7 +851,7 @@ namespace hfst {
for (StringSet::iterator it = unknown.begin();
it != unknown.end(); it++)
{
- if (not FdOperation::is_diacritic(*it)) {
+ if (! FdOperation::is_diacritic(*it)) {
int64 number = is->Find(*it);
result->AddArc(result_s,
StdArc(number, arc.olabel,
@@ -864,7 +864,7 @@ namespace hfst {
for (StringSet::iterator it = unknown.begin();
it != unknown.end(); it++)
{
- if (not FdOperation::is_diacritic(*it)) {
+ if (! FdOperation::is_diacritic(*it)) {
int64 number = is->Find(*it);
result->AddArc(result_s,
StdArc(arc.ilabel, number,
@@ -889,12 +889,12 @@ namespace hfst {
StdVectorFst * result = new StdVectorFst();
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
result->AddState();
// go through all states in this
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
// create new state in result, if needed
StateId s = siter.Value();
@@ -931,7 +931,7 @@ namespace hfst {
{
unsigned int retval=0;
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
retval++;
return retval;
}
@@ -940,7 +940,7 @@ namespace hfst {
{
unsigned int retval=0;
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
for (fst::ArcIterator<StdVectorFst> aiter(*t,siter.Value());
!aiter.Done(); aiter.Next())
@@ -1013,14 +1013,14 @@ namespace hfst {
fst::StdVectorFst *harmonized_t1;
fst::StdVectorFst *harmonized_t2;
- if (not unknown_symbols_in_use)
+ if (! unknown_symbols_in_use)
harmonized_t1 = t1;
else {
harmonized_t1 = expand_arcs(t1, unknown_t1, unknown_symbols_in_use);
harmonized_t1->SetInputSymbols(t1->InputSymbols());
}
- if (not unknown_symbols_in_use)
+ if (! unknown_symbols_in_use)
harmonized_t2 = t2;
else {
harmonized_t2 = expand_arcs(t2, unknown_t2, unknown_symbols_in_use);
@@ -1284,7 +1284,7 @@ namespace hfst {
(const std::string &symbol)
{
- assert(not (symbol == ""));
+ assert(! (symbol == ""));
StdVectorFst * t = new StdVectorFst;
SymbolTable st = create_symbol_table("");
@@ -1303,8 +1303,8 @@ namespace hfst {
(const std::string &isymbol, const std::string &osymbol)
{
- assert(not (isymbol == ""));
- assert(not (osymbol == ""));
+ assert(! (isymbol == ""));
+ assert(! (osymbol == ""));
StdVectorFst * t = new StdVectorFst;
SymbolTable st = create_symbol_table("");
@@ -1354,7 +1354,7 @@ namespace hfst {
bool TropicalWeightTransducer::is_automaton(StdVectorFst * t)
{
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::ArcIterator<StdVectorFst> aiter(*t,s);
@@ -1390,8 +1390,8 @@ namespace hfst {
{
StateId s2 = t->AddState();
- assert(not (it->first == ""));
- assert(not (it->second == ""));
+ assert(! (it->first == ""));
+ assert(! (it->second == ""));
t->AddArc(s1,StdArc(st.AddSymbol(it->first),
st.AddSymbol(it->second),0,s2));
@@ -1412,15 +1412,15 @@ namespace hfst {
t->SetStart(s1);
StateId s2=s1; // final state
- if (not sps.empty()) {
- if (not cyclic)
+ if (! sps.empty()) {
+ if (! cyclic)
s2 = t->AddState();
for (StringPairSet::const_iterator it = sps.begin();
it != sps.end();
++it)
{
- assert(not (it->first == ""));
- assert(not (it->second == ""));
+ assert(! (it->first == ""));
+ assert(! (it->second == ""));
t->AddArc(s1,StdArc(st.AddSymbol(it->first),
st.AddSymbol(it->second),0,s2));
@@ -1447,8 +1447,8 @@ namespace hfst {
for (StringPairSet::const_iterator it2 = (*it).begin();
it2 != (*it).end(); it2++ ) {
- assert(not (it2->first == ""));
- assert(not (it2->second == ""));
+ assert(! (it2->first == ""));
+ assert(! (it2->second == ""));
t->AddArc(s1,StdArc(st.AddSymbol(it2->first),
st.AddSymbol(it2->second),0,s2));
@@ -1489,8 +1489,8 @@ namespace hfst {
t->SetStart(s1);
StateId s2=s1; // final state
- if (not nps.empty()) {
- if (not cyclic)
+ if (! nps.empty()) {
+ if (! cyclic)
s2 = t->AddState();
for (NumberPairSet::const_iterator it = nps.begin();
it != nps.end();
@@ -1577,7 +1577,7 @@ namespace hfst {
void print_att_number(StdVectorFst *t, FILE * ofile) {
fprintf(ofile, "initial state: %i\n", t->Start());
for (fst::StateIterator<StdVectorFst> siter(*t);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
if ( t->Final(s) != TropicalWeight::Zero() )
@@ -1878,7 +1878,7 @@ namespace hfst {
fst::SymbolTable * st = tc->InputSymbols()->Copy();
assert(st != NULL);
for (fst::StateIterator<fst::StdVectorFst> siter(*tc);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::MutableArcIterator<StdVectorFst> aiter(tc,s);
@@ -1936,12 +1936,13 @@ namespace hfst {
{
assert(t->InputSymbols() != NULL);
SymbolTable * st = t->InputSymbols()->Copy();
+ // TODO: cl.exe: conversion from 'int64' to 'unsigned int'
pair<unsigned int, unsigned int> old_pair
- (st->AddSymbol(old_symbol_pair.first),
- st->AddSymbol(old_symbol_pair.second));
+ ((unsigned int)st->AddSymbol(old_symbol_pair.first),
+ (unsigned int)st->AddSymbol(old_symbol_pair.second));
pair<unsigned int, unsigned int> new_pair
- (st->AddSymbol(new_symbol_pair.first),
- st->AddSymbol(new_symbol_pair.second));
+ ((unsigned int)st->AddSymbol(new_symbol_pair.first),
+ (unsigned int)st->AddSymbol(new_symbol_pair.second));
StdVectorFst * retval = substitute(t, old_pair, new_pair);
retval->SetInputSymbols(st);
delete st;
@@ -1961,7 +1962,7 @@ namespace hfst {
for( int i = 0; i < states; ++i ) {
for (fst::MutableArcIterator<fst::StdVectorFst> it(t,i);
- not it.Done();
+ ! it.Done();
it.Next()) {
fst::StdArc arc = it.Value();
@@ -2047,7 +2048,7 @@ namespace hfst {
for( unsigned int i = 0; i < (unsigned int)states; ++i ) {
for (fst::MutableArcIterator<fst::StdVectorFst> it(t,i);
- not it.Done();
+ ! it.Done();
it.Next()) {
fst::StdArc arc = it.Value();
@@ -2188,7 +2189,7 @@ namespace hfst {
}
}
- if (not transition_found) {
+ if (! transition_found) {
StateId new_state = t->AddState();
t->AddArc(s, StdArc(inumber,onumber,0,new_state));
s = new_state;
@@ -2223,7 +2224,7 @@ namespace hfst {
}
}
- if (not transition_found) {
+ if (! transition_found) {
StateId new_state = t->AddState();
t->AddArc(s, StdArc(inumber,onumber,0,new_state));
s = new_state;
@@ -2289,7 +2290,7 @@ namespace hfst {
StdVectorFst *t2_ = copy(t2);
for (fst::StateIterator<StdVectorFst> siter(*t2_);
- not siter.Done(); siter.Next())
+ ! siter.Done(); siter.Next())
{
StateId s = siter.Value();
for (fst::MutableArcIterator<StdVectorFst> aiter(t2_,s);
@@ -2340,7 +2341,7 @@ namespace hfst {
{
StdVectorFst * t_copy = new StdVectorFst(*t);
for (fst::StateIterator<StdVectorFst> iter(*t);
- not iter.Done(); iter.Next())
+ ! iter.Done(); iter.Next())
{
if (t_copy->Final(iter.Value()) != fst::TropicalWeight::Zero())
{ t_copy->SetFinal(iter.Value(),f); }
@@ -2360,7 +2361,7 @@ namespace hfst {
!aiter.Done();
aiter.Next())
{
- if ((aiter.Value().ilabel == ilabel) and
+ if ((aiter.Value().ilabel == ilabel) &&
(aiter.Value().olabel == olabel))
{ return aiter.Position(); }
}
@@ -2637,7 +2638,7 @@ namespace hfst {
/* Go through all transitions in a random order.
(If \a t is pruned, only one transition is proceeded.) */
- while ( not t_transitions.empty() ) {
+ while ( ! t_transitions.empty() ) {
unsigned int index = rand() % t_transitions.size();
fst::StdArc arc = t_transitions.at(index);
t_transitions.erase(t_transitions.begin()+index);
@@ -2854,7 +2855,7 @@ namespace hfst {
}
fst::SymbolTable *output_st=NULL;
- if (not hfst_format) {
+ if (! hfst_format) {
output_st = new fst::SymbolTable(*(transducer->InputSymbols()));
transducer->SetOutputSymbols(output_st);
}
diff --git a/libhfst/src/implementations/TropicalWeightTransducer.h b/libhfst/src/implementations/TropicalWeightTransducer.h
index dbb02a3..8080821 100644
--- a/libhfst/src/implementations/TropicalWeightTransducer.h
+++ b/libhfst/src/implementations/TropicalWeightTransducer.h
@@ -16,12 +16,15 @@
#include "HfstExceptionDefs.h"
#include "HfstFlagDiacritics.h"
-#ifdef WINDOWS
+#if HAVE_CONFIG_H
+ #include "../../../config.h"
+#endif
+
+#ifdef _MSC_VER
#include "back-ends/openfstwin/src/include/fst/fstlib.h"
#else
-#include "../../../config.h"
#include "back-ends/openfst/src/include/fst/fstlib.h"
-#endif // WINDOWS
+#endif // _MSC_VER
#include "HfstExtractStrings.h"
#include <cstdio>
diff --git a/libhfst/src/implementations/XfsmTransducer.cc b/libhfst/src/implementations/XfsmTransducer.cc
new file mode 100644
index 0000000..4b4aa62
--- /dev/null
+++ b/libhfst/src/implementations/XfsmTransducer.cc
@@ -0,0 +1,555 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include "XfsmTransducer.h"
+#include <cstring>
+
+#ifndef MAIN_TEST
+namespace hfst { namespace implementations {
+
+ void XfsmTransducer::initialize_xfsm()
+ {
+ FST_CNTXTptr cntxt = initialize_cfsm();
+ (void) set_char_encoding(cntxt, CHAR_ENC_UTF_8);
+ IY_VERBOSE = 0; // suppress messages
+ }
+
+
+ // ---------- XfsmInputStream functions ----------
+
+
+ /** Create an XfsmInputStream that reads from stdin. */
+ XfsmInputStream::XfsmInputStream(void)
+ {
+ HFST_THROW_MESSAGE
+ (FunctionNotImplementedException,
+ "XfsmInputStream::XfsmInputStream() not supported");
+ }
+
+ /** Create an XfsmInputStream that reads from file \a filename. */
+ XfsmInputStream::XfsmInputStream(const std::string &filename_):
+ filename(std::string(filename_)), net_list(NULL), list_size(-1), list_pos(-1)
+ {
+ if (filename == std::string())
+ { HFST_THROW_MESSAGE
+ (FunctionNotImplementedException,
+ "XfsmInputStream::XfsmInputStream(\"\") not supported");
+ }
+ else {
+ FILE * input_file = fopen(filename.c_str(),"r");
+ if (input_file == NULL)
+ {
+ HFST_THROW(StreamNotReadableException); }
+ fclose(input_file);
+
+ char * fn = strdup(filename.c_str());
+ FST_CNTXTptr fst_cntxt = get_default_cfsm_context();
+ net_list = load_nets(fn, fst_cntxt);
+ free(fn);
+ if (net_list == NULL)
+ { HFST_THROW(StreamNotReadableException); }
+ list_size = NV_len(net_list);
+ if (list_size <= 0)
+ { HFST_THROW(HfstFatalException); }
+ list_pos = 0;
+ }
+ }
+
+ /** Close the stream. */
+ void XfsmInputStream::close(void)
+ {
+ free_nv_and_nets(net_list);
+ list_size = -1;
+ list_pos = -1;
+ return;
+ }
+
+ bool XfsmInputStream::is_eof(void)
+ {
+ return (list_pos >= list_size);
+ }
+
+ bool XfsmInputStream::is_bad(void)
+ {
+ return is_eof();
+ }
+
+ bool XfsmInputStream::is_good(void)
+ {
+ return not is_bad();
+ };
+
+ NETptr XfsmInputStream::read_transducer()
+ {
+ if (is_eof())
+ return NULL;
+ NETptr retval = nv_get(net_list, list_pos);
+ if (retval == NULL)
+ { HFST_THROW(StreamNotReadableException); }
+ ++list_pos;
+ return XfsmTransducer::copy(retval);
+ };
+
+
+ // ---------- XfsmOutputStream functions ----------
+
+ XfsmOutputStream::XfsmOutputStream(void)
+ {
+ throw "XfsmOutputStream::XfsmOutputStream() not supported";
+ }
+
+ XfsmOutputStream::XfsmOutputStream(const std::string &str):
+ filename(std::string(str)), net_list(NULL)
+ {
+ if (filename != std::string()) {
+ FILE * ofile = fopen(filename.c_str(), "wb");
+ if (ofile == NULL) {
+ HFST_THROW(StreamNotReadableException);
+ }
+ fclose(ofile); /* XFSM api only has a function save_net(char * filename). */
+ }
+ else {
+ throw "XfsmOutputStream::XfsmOutputStream(\"\") not supported";
+ }
+ }
+
+ void XfsmOutputStream::close(void)
+ {
+ /* ofile is closed already as we use filenames. */
+ }
+
+ void XfsmOutputStream::flush()
+ {
+ if (net_list != NULL)
+ {
+ FST_CNTXTptr cptr = get_default_cfsm_context();
+ char * fn = strdup(filename.c_str());
+ if (save_nets(net_list, fn, cptr) != 0)
+ HFST_THROW_MESSAGE
+ (HfstFatalException,
+ "an error happened when writing an xfsm transducer");
+ free(fn);
+ }
+ free_nv_and_nets(net_list);
+ net_list = NULL;
+ }
+
+ /* Writing is delayed and done when flush() is called. */
+ void XfsmOutputStream::write_transducer(NETptr transducer)
+ {
+ if (net_list == NULL)
+ {
+ net_list = make_nv(0);
+ }
+ nv_add(XfsmTransducer::copy(transducer), net_list);
+ }
+
+ static id_type hfst_symbol_to_xfsm_symbol(const std::string & symbol);
+ static std::string xfsm_symbol_to_hfst_symbol(id_type id);
+ static void label_id_to_symbol_pair(id_type label_id, std::string & isymbol, std::string & osymbol);
+ static id_type symbol_pair_to_label_id(const std::string & isymbol, const std::string & osymbol);
+
+ // Convert between hfst and xfsm one-side symbols.
+ // The identity symbol must be handled separately.
+ id_type XfsmTransducer::hfst_symbol_to_xfsm_symbol(const std::string & symbol)
+ {
+ if (symbol == hfst::internal_epsilon)
+ return EPSILON;
+ else if (symbol == hfst::internal_unknown)
+ return OTHER;
+ else if (symbol == hfst::internal_identity)
+ throw "hfst_symbol_to_xfsm_symbol does not accept the identity symbol as its argument";
+ else
+ return single_to_id(symbol.c_str());
+ }
+
+ // Convert between xfsm and hfst one-side symbols.
+ // The identity symbol must be handled separately.
+ std::string XfsmTransducer::xfsm_symbol_to_hfst_symbol(id_type id)
+ {
+ if (id == EPSILON)
+ return hfst::internal_epsilon;
+ else if (id == OTHER)
+ return hfst::internal_unknown;
+ else {
+ std::string retval("");
+ LABELptr lptr = id_to_label(id);
+ FAT_STR fs = lptr->content.name;
+ while (*fs != '\0')
+ {
+ retval.append(1, *fs);
+ ++fs;
+ }
+ return retval;
+ }
+ }
+
+ // Convert between an xfsm label (symbol pair) and hfst transition symbols.
+ void XfsmTransducer::label_id_to_symbol_pair(id_type label_id, std::string & isymbol, std::string & osymbol)
+ {
+ // (1) atomic OTHER label -> identity pair
+ if (label_id == OTHER)
+ {
+ isymbol = hfst::internal_identity;
+ osymbol = hfst::internal_identity;
+ }
+ else
+ {
+ // (2) non-atomic OTHER label -> unknown pair
+ // (3) all other cases
+ id_type upperid = upper_id(label_id);
+ id_type lowerid = lower_id(label_id);
+ isymbol = xfsm_symbol_to_hfst_symbol(upperid);
+ osymbol = xfsm_symbol_to_hfst_symbol(lowerid);
+ }
+ }
+
+ id_type XfsmTransducer::symbol_pair_to_label_id(const std::string & isymbol, const std::string & osymbol)
+ {
+ if (isymbol == hfst::internal_identity)
+ {
+ if (osymbol != hfst::internal_identity)
+ throw "identity symbol cannot be on one side only";
+ // atomic OTHER label
+ return OTHER;
+ }
+ else
+ {
+ id_type input_id = hfst_symbol_to_xfsm_symbol(isymbol);
+ id_type output_id = hfst_symbol_to_xfsm_symbol(osymbol);
+ return id_pair_to_id(input_id, output_id);
+ }
+ }
+
+ NETptr XfsmTransducer::create_xfsm_unknown_to_unknown_transducer()
+ {
+ NETptr result = null_net();
+ STATEptr final = add_state_to_net(result, 1);
+ id_type ti = id_pair_to_id(OTHER, OTHER);
+ if( add_arc_to_state(result, result->start.state, ti, final, NULL, 0) == NULL )
+ throw "add_arc_to_state failed";
+ return result;
+ }
+
+ NETptr XfsmTransducer::create_xfsm_identity_to_identity_transducer()
+ {
+ NETptr result = null_net();
+ STATEptr final = add_state_to_net(result, 1);
+ id_type ti = OTHER;
+ if( add_arc_to_state(result, result->start.state, ti, final, NULL, 0) == NULL )
+ throw "add_arc_to_state failed";
+ return result;
+ }
+
+ NETptr XfsmTransducer::create_empty_transducer(void)
+ {
+ return null_net();
+ }
+
+ NETptr XfsmTransducer::create_epsilon_transducer(void)
+ {
+ NETptr result = null_net();
+ return optional_net(result, DONT_KEEP);
+ }
+
+ //NETptr read_regex(const char *regex_str);
+
+ NETptr XfsmTransducer::define_transducer(const hfst::StringPairVector &spv)
+ {
+ std::string regex("");
+ for (hfst::StringPairVector::const_iterator it = spv.begin(); it != spv.end(); it++)
+ {
+ regex = regex + "\"" + it->first + "\":\"" + it->second + "\" ";
+ }
+ return read_regex(regex.c_str());
+ }
+
+ NETptr XfsmTransducer::define_transducer(const hfst::StringPairSet &sps, bool cyclic/*=false*/)
+ {
+ std::string regex("");
+ for (hfst::StringPairSet::const_iterator it = sps.begin(); it != sps.end(); it++)
+ {
+ if (it != sps.begin())
+ regex = regex + " | ";
+ regex = regex + "\"" + it->first + "\":\"" + it->second + "\"";
+ }
+ if (cyclic)
+ regex = "[" + regex + "]*";
+ return read_regex(regex.c_str());
+ }
+
+ NETptr XfsmTransducer::define_transducer(const std::vector<StringPairSet> &spsv)
+ {
+ std::string regex("");
+ for (std::vector<StringPairSet>::const_iterator it1 = spsv.begin(); it1 != spsv.end(); it1++)
+ {
+ regex = regex + "[";
+ for (hfst::StringPairSet::const_iterator it2 = it1->begin(); it2 != it1->end(); it2++)
+ {
+ if (it2 != it1->begin())
+ regex = regex + " | ";
+ regex = regex + "\"" + it2->first + "\":\"" + it2->second + "\"";
+ }
+ regex = regex + "] ";
+ }
+ return read_regex(regex.c_str());
+ }
+
+ NETptr XfsmTransducer::define_transducer(const std::string &symbol)
+ {
+ std::string regex = "\"" + symbol + "\"";
+ return read_regex(regex.c_str());
+ }
+
+ NETptr XfsmTransducer::define_transducer(const std::string &isymbol, const std::string &osymbol)
+ {
+ std::string regex = "\"" + isymbol + "\":\"" + osymbol + "\"";
+ return read_regex(regex.c_str());
+ }
+
+ NETptr XfsmTransducer::copy(NETptr t)
+ {
+ return copy_net(t);
+ }
+
+ void XfsmTransducer::set_compose_flag_as_special(bool value)
+ {
+ FST_CNTXTptr fst_cntxt = get_default_cfsm_context();
+ fst_cntxt->interface->general.compose_flag_as_special = (value) ? 1 : 0 ;
+ }
+
+ bool XfsmTransducer::minimize_even_if_already_minimal_ = false;
+
+ void XfsmTransducer::set_minimize_even_if_already_minimal(bool value)
+ {
+ minimize_even_if_already_minimal_ = value;
+ }
+
+ NETptr XfsmTransducer::minimize(NETptr t)
+ {
+ if (minimize_even_if_already_minimal_)
+ {
+ NET_minimized(t) = 0;
+ }
+ if (minimize_net(t) == 1)
+ {
+ HFST_THROW(HfstFatalException);
+ }
+ return t;
+ }
+
+ void XfsmTransducer::add_symbol_to_alphabet(NETptr t, const std::string & symbol)
+ {
+ ALPHABETptr ap = net_sigma(t);
+ if (hfst::is_epsilon(symbol) || hfst::is_unknown(symbol) || hfst::is_identity(symbol))
+ return;
+ (void) alph_add_to(ap, XfsmTransducer::hfst_symbol_to_xfsm_symbol(symbol.c_str()), DONT_KEEP);
+ }
+
+ void XfsmTransducer::add_symbols_to_alphabet(NETptr t, const StringSet & symbols)
+ {
+ ALPHABETptr ap = net_sigma(t);
+
+ for (StringSet::const_iterator it = symbols.begin(); it != symbols.end(); it++)
+ {
+ if (hfst::is_epsilon(*it) || hfst::is_unknown(*it) || hfst::is_identity(*it))
+ continue;
+ (void) alph_add_to(ap, XfsmTransducer::hfst_symbol_to_xfsm_symbol(it->c_str()), DONT_KEEP);
+ }
+ }
+
+ void XfsmTransducer::remove_symbols_from_alphabet(NETptr t, const StringSet & symbols)
+ {
+ ALPHABETptr ap = net_sigma(t);
+
+ for (StringSet::const_iterator it = symbols.begin(); it != symbols.end(); it++)
+ {
+ if (hfst::is_epsilon(*it) || hfst::is_unknown(*it) || hfst::is_identity(*it))
+ continue;
+ (void)alph_remove_from(ap, XfsmTransducer::hfst_symbol_to_xfsm_symbol(it->c_str()), DONT_KEEP);
+ }
+ }
+
+ StringSet XfsmTransducer::get_alphabet(const NETptr t)
+ {
+ StringSet retval;
+ ALPHABETptr alpha_ptr = net_sigma(const_cast<NETptr>(t));
+ ALPH_ITptr alpha_it_ptr = start_alph_iterator(NULL, alpha_ptr);
+ id_type label_id = next_alph_id(alpha_it_ptr);
+
+ while(label_id != ID_NO_SYMBOL)
+ {
+ std::string symbol = XfsmTransducer::xfsm_symbol_to_hfst_symbol(label_id);
+ retval.insert(symbol);
+ label_id = next_alph_id(alpha_it_ptr);
+ }
+ return retval;
+ }
+
+ NETptr XfsmTransducer::compose(NETptr t1, const NETptr t2)
+ {
+ return compose_net(t1, const_cast<NETptr>(t2), DONT_KEEP, KEEP);
+ }
+
+ NETptr XfsmTransducer::concatenate(NETptr t1, const NETptr t2)
+ {
+ return concat_net(t1, const_cast<NETptr>(t2), DONT_KEEP, KEEP);
+ }
+
+ NETptr XfsmTransducer::disjunct(NETptr t1, const NETptr t2)
+ {
+ return union_net(t1, const_cast<NETptr>(t2), DONT_KEEP, KEEP);
+ }
+
+ NETptr XfsmTransducer::intersect(NETptr t1, const NETptr t2)
+ {
+ return intersect_net(t1, const_cast<NETptr>(t2), DONT_KEEP, KEEP);
+ }
+
+ NETptr XfsmTransducer::subtract(NETptr t1, const NETptr t2)
+ {
+ return minus_net(t1, const_cast<NETptr>(t2), DONT_KEEP, KEEP);
+ }
+
+ bool XfsmTransducer::are_equivalent(NETptr t1, NETptr t2)
+ {
+ return (test_equivalent(t1, t2) == 1);
+ }
+
+ bool XfsmTransducer::is_cyclic(NETptr t)
+ {
+ if (test_upper_bounded(t) != 1 || test_lower_bounded(t) != 1)
+ return true;
+ return false;
+ }
+
+ unsigned int XfsmTransducer::number_of_states(NETptr t)
+ {
+ return (unsigned int) NET_num_states(t);
+ }
+
+ unsigned int XfsmTransducer::number_of_arcs(NETptr t)
+ {
+ return (unsigned int) NET_num_arcs(t);
+ }
+
+ NETptr XfsmTransducer::eliminate_flags_xfsm(NETptr t)
+ {
+ return eliminate_flag(t, NULL, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::eliminate_flag_xfsm(NETptr t, const std::string & flag)
+ {
+ char * f = strdup(flag.c_str());
+ NETptr retval = eliminate_flag(t, f, DONT_KEEP);
+ free(f);
+ return retval;
+ }
+
+ NETptr XfsmTransducer::repeat_star(NETptr t)
+ {
+ return repeat_net(t, 0, -1, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::repeat_plus(NETptr t)
+ {
+ return repeat_net(t, 1, -1, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::repeat_n(NETptr t, unsigned int n)
+ {
+ return repeat_net(t, n, n, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::repeat_le_n(NETptr t, unsigned int n)
+ {
+ return repeat_net(t, 0, n, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::repeat_n_plus(NETptr t, unsigned int n)
+ {
+ return repeat_net(t, n, -1, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::repeat_n_to_k(NETptr t, unsigned int n, unsigned int k)
+ {
+ return repeat_net(t, n, k, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::optionalize(NETptr t)
+ {
+ return optional_net(t, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::invert(NETptr t)
+ {
+ return invert_net(t, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::reverse(NETptr t)
+ {
+ return reverse_net(t, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::extract_input_language(NETptr t)
+ {
+ return upper_side_net(t, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::extract_output_language(NETptr t)
+ {
+ return lower_side_net(t, DONT_KEEP);
+ }
+
+ NETptr XfsmTransducer::insert_freely(NETptr t, NETptr ins)
+ {
+ return ignore_net(t, const_cast<NETptr>(ins), DONT_KEEP, KEEP);
+ }
+
+ void XfsmTransducer::write_in_att_format(NETptr t, const char * filename)
+ {
+ HFST_THROW(HfstFatalException);
+ }
+
+ void XfsmTransducer::write_in_prolog_format(NETptr t, const char * filename)
+ {
+ char * f = strdup(filename);
+ if (write_prolog(t, f) != 0)
+ HFST_THROW(HfstFatalException);
+ free(f);
+ }
+
+ NETptr XfsmTransducer::prolog_file_to_xfsm_transducer(const char * filename)
+ {
+ char * f = strdup(filename);
+ NETptr retval = read_prolog(f);
+ if (retval == NULL)
+ HFST_THROW(HfstFatalException);
+ free(f);
+ return retval;
+ }
+
+ } }
+
+#else // MAIN_TEST was defined
+#include <cstdlib>
+#include <cassert>
+#include <iostream>
+using namespace hfst::implementations;
+
+int main(int argc, char * argv[])
+{
+ std::cout << "Unit tests for " __FILE__ ":";
+ std::cout << std::endl << "ok" << std::endl;
+ return EXIT_SUCCESS;
+}
+#endif // MAIN_TEST
diff --git a/libhfst/src/implementations/XfsmTransducer.h b/libhfst/src/implementations/XfsmTransducer.h
new file mode 100644
index 0000000..6d3dfd3
--- /dev/null
+++ b/libhfst/src/implementations/XfsmTransducer.h
@@ -0,0 +1,164 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef _XFSM_TRANSDUCER_H_
+#define _XFSM_TRANSDUCER_H_
+
+#include "HfstSymbolDefs.h"
+#include "HfstExceptionDefs.h"
+#include "HfstExtractStrings.h"
+#include "HfstFlagDiacritics.h"
+#include <stdlib.h>
+
+#ifndef _XFSMLIB_H_
+#define _XFSMLIB_H_
+#include "xfsm/xfsm_api.h"
+#endif
+#include <cstdio>
+#include <string>
+#include <sstream>
+#include <iostream>
+
+#include "../FormatSpecifiers.h"
+
+/** @file XfsmTransducer.h
+ \brief Declarations of functions and datatypes that form a bridge between
+ HFST API and xfsm. */
+
+namespace hfst {
+ namespace implementations
+{
+
+ /* A class for reading XFSM binary transducers from a file. */
+ class XfsmInputStream
+ {
+ private:
+ /* The name of the file where transducers will be read from. */
+ std::string filename;
+ /* A list of transducers that will return transducers one by one when
+ read_transducer() is called. */
+ NVptr net_list;
+ /* The size of net_list. */
+ int list_size;
+ /* The position of the transducer that read_transducer() will return. */
+ int list_pos;
+ public:
+ /* Throws an error. */
+ XfsmInputStream(void);
+ /* A stream that will read transducers from file \a filename. */
+ XfsmInputStream(const std::string &filename);
+ /* Frees the memory allocated for this XfsmInputStream. */
+ void close(void);
+ /* Whether there are any transducers left in net_list. */
+ bool is_eof(void);
+ /* Whether the stream is bad for reading, always false. */
+ bool is_bad(void);
+ /* Whether the stream is good for reading, always true. */
+ bool is_good(void);
+ /* Whether the next item returned by read_transducer() is a valid
+ XFSM transducer. Basically always true, except if is_eof() is true. */
+ bool is_fst(void);
+ /* Delayed read, returns a transducer from net_list. */
+ NETptr read_transducer();
+ };
+
+ /* A class for writing XFSM transducers in binary format to a file. */
+ class XfsmOutputStream
+ {
+ private:
+ /* The name of the file where transducers will be written. */
+ std::string filename;
+ /* A list of transducers to be written when flush() is called. */
+ NVptr net_list;
+ public:
+ /* Throws an error. */
+ XfsmOutputStream(void);
+ /* A stream that will write transducers into file \a filename. */
+ XfsmOutputStream(const std::string &filename);
+ /* Writes the contents of net_list into file filename. */
+ void flush();
+ /* Does nothing. */
+ void close();
+ /* Delayed write, stores a copy of \a transducer in net_list. */
+ void write_transducer(NETptr transducer);
+ };
+
+ class XfsmTransducer {
+ private:
+ static bool minimize_even_if_already_minimal_;
+
+ public:
+ static void initialize_xfsm();
+ static NETptr create_xfsm_unknown_to_unknown_transducer();
+ static NETptr create_xfsm_identity_to_identity_transducer();
+ static id_type hfst_symbol_to_xfsm_symbol(const std::string & symbol);
+ static std::string xfsm_symbol_to_hfst_symbol(id_type id);
+ static void label_id_to_symbol_pair(id_type label_id, std::string & isymbol, std::string & osymbol);
+ static id_type symbol_pair_to_label_id(const std::string & isymbol, const std::string & osymbol);
+
+ static NETptr create_empty_transducer(void);
+ static NETptr create_epsilon_transducer(void);
+ static NETptr define_transducer(const hfst::StringPairVector &spv);
+ static NETptr define_transducer
+ (const hfst::StringPairSet &sps, bool cyclic=false);
+ static NETptr define_transducer(const std::vector<StringPairSet> &spsv);
+ static NETptr define_transducer
+ (const std::string &symbol);
+ static NETptr define_transducer
+ (const std::string &isymbol, const std::string &osymbol);
+ static NETptr copy(NETptr t);
+ static NETptr minimize(NETptr t);
+
+ static NETptr compose(NETptr t1, const NETptr t2);
+ static NETptr concatenate(NETptr t1, const NETptr t2);
+ static NETptr disjunct(NETptr t1, const NETptr t2);
+ static NETptr intersect(NETptr t1, const NETptr t2);
+ static NETptr subtract(NETptr t1, const NETptr t2);
+
+ static bool are_equivalent(NETptr t1, NETptr t2);
+ static bool is_cyclic(NETptr t);
+ static unsigned int number_of_states(NETptr t);
+ static unsigned int number_of_arcs(NETptr t);
+ static NETptr eliminate_flags_xfsm(NETptr t);
+ static NETptr eliminate_flag_xfsm(NETptr t, const std::string & flag);
+
+ static NETptr repeat_star(NETptr t);
+ static NETptr repeat_plus(NETptr t);
+ static NETptr repeat_n(NETptr t, unsigned int n);
+ static NETptr repeat_le_n(NETptr t, unsigned int n);
+ static NETptr repeat_n_plus(NETptr t, unsigned int n);
+ static NETptr repeat_n_to_k(NETptr t, unsigned int n, unsigned int k);
+
+ static NETptr optionalize(NETptr t);
+ static NETptr invert(NETptr t);
+ static NETptr reverse(NETptr t);
+ static NETptr extract_input_language(NETptr t);
+ static NETptr extract_output_language(NETptr t);
+
+ static NETptr insert_freely(NETptr t, const NETptr ins);
+
+ static StringSet get_alphabet(const NETptr t);
+ static void add_symbol_to_alphabet(NETptr t, const std::string & symbol);
+ static void add_symbols_to_alphabet(NETptr t, const StringSet & symbols);
+ static void remove_symbols_from_alphabet(NETptr t, const StringSet & symbols);
+
+ // for debugging
+ static void set_minimize_even_if_already_minimal(bool value);
+ static void set_compose_flag_as_special(bool value);
+
+ static void write_in_att_format(NETptr t, const char * filename);
+ static void write_in_prolog_format(NETptr t, const char * filename);
+ static NETptr prolog_file_to_xfsm_transducer(const char * filename);
+
+ } ;
+} }
+#endif
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc
index 6bc73dd..de50547 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.cc
@@ -9,7 +9,7 @@ namespace hfst
namespace implementations
{
- const HfstState ComposeIntersectFst::START;
+ const HfstState ComposeIntersectFst::START = 0;
template<> ComposeIntersectFst::CompareTransitions
ComposeIntersectFst::TransitionSet::comparator =
@@ -39,9 +39,9 @@ namespace hfst
(const ComposeIntersectFst::Transition &another) const
{
return
- ilabel == another.ilabel and
- olabel == another.olabel and
- weight == another.weight and
+ ilabel == another.ilabel &&
+ olabel == another.olabel &&
+ weight == another.weight &&
target == another.target;
}
bool ComposeIntersectFst::CompareTransitions::operator()
@@ -115,7 +115,7 @@ namespace hfst
(jt->get_output_symbol())].
insert(*jt); }
}
- if (not identity_found)
+ if (! identity_found)
{ identity_transition_vector.push_back
(Transition
(0,
@@ -149,7 +149,7 @@ namespace hfst
if (transition_map_vector.at(s).find(symbol) ==
transition_map_vector.at(s).end())
{
- if (is_known_symbol(symbol) or not has_identity_transition(s))
+ if (is_known_symbol(symbol) || ! has_identity_transition(s))
{ return transition_map_vector.at(s)[symbol] = TransitionSet(); }
else
{
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h
index 161558b..de9d3ed 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectFst.h
@@ -19,13 +19,13 @@ namespace hfst
public:
struct Transition
{
- size_t ilabel;
- size_t olabel;
- float weight;
- HfstState target;
- Transition(const HfstBasicTransition &);
- Transition(HfstState,size_t,size_t,float);
- bool operator==(const Transition&) const;
+ size_t ilabel;
+ size_t olabel;
+ float weight;
+ HfstState target;
+ Transition(const HfstBasicTransition &);
+ Transition(HfstState,size_t,size_t,float);
+ bool operator==(const Transition&) const;
};
struct CompareTransitions
@@ -38,7 +38,7 @@ namespace hfst
<Transition,CompareTransitions>
TransitionSet;
typedef std::set<size_t> SymbolSet;
- static const HfstState START = 0;
+ static const HfstState START; // = 0;
ComposeIntersectFst(const HfstBasicTransducer &, bool input_keys);
ComposeIntersectFst(void);
virtual ~ComposeIntersectFst(void);
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc
index 7fa76e7..dd0806c 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.cc
@@ -24,18 +24,18 @@ namespace hfst
state_pair_map.clear();
pair_vector.clear();
- while (not agenda.empty())
+ while (! agenda.empty())
{ agenda.pop(); }
result = HfstBasicTransducer();
}
HfstState ComposeIntersectLexicon::map_state_and_add_to_agenda
- (const StatePair &p)
+ (const StatePair &p, bool allow_lexicon_epsilons)
{
HfstState s;
- if (p.first == START and p.second == ComposeIntersectRule::START)
+ if (p.first == START && p.second == ComposeIntersectRule::START)
{ s = 0; }
else
{ s = result.add_state(); }
@@ -46,9 +46,14 @@ namespace hfst
state_pair_map[p] = s;
pair_vector.push_back(p);
agenda.push(s);
+ lexicon_non_epsilon_states.insert(s);
+
return s;
}
+ bool ComposeIntersectLexicon::can_have_lexicon_epsilons(HfstState s)
+ { return lexicon_non_epsilon_states.count(s) > 0; }
+
HfstBasicTransducer ComposeIntersectLexicon::compose_with_rules
(ComposeIntersectRule * rules)
{
@@ -56,15 +61,18 @@ namespace hfst
StatePair start_pair = StatePair(START,ComposeIntersectRule::START);
// This will return 0.
- (void)map_state_and_add_to_agenda(start_pair);
+ (void)map_state_and_add_to_agenda(start_pair, true);
return compute_composition_result(rules);
}
- HfstState ComposeIntersectLexicon::get_state(const StatePair &p)
+ HfstState ComposeIntersectLexicon::get_state(const StatePair &p,
+ bool allow_lexicon_epsilons)
{
if (state_pair_map.find(p) == state_pair_map.end())
- { return map_state_and_add_to_agenda(p); }
+ {
+ return map_state_and_add_to_agenda(p, allow_lexicon_epsilons);
+ }
return state_pair_map[p];
}
@@ -76,7 +84,7 @@ namespace hfst
{
float lexicon_weight = get_final_weight(pair_vector[s].first);
float rules_weight = rules->get_final_weight(pair_vector[s].second);
- if (lexicon_weight != std::numeric_limits<float>::infinity() and
+ if (lexicon_weight != std::numeric_limits<float>::infinity() &&
rules_weight != std::numeric_limits<float>::infinity())
{ result.set_final_weight(s,lexicon_weight+rules_weight); }
}
@@ -85,13 +93,14 @@ namespace hfst
HfstBasicTransducer &ComposeIntersectLexicon::compute_composition_result
(ComposeIntersectRule * rules)
{
- while (not agenda.empty())
- {
- HfstState s = agenda.front();
- agenda.pop();
- compute_state(s,rules);
- }
-
+ while (! agenda.empty())
+ {
+ HfstState s = agenda.front();
+ agenda.pop();
+
+ compute_state(s, rules, can_have_lexicon_epsilons(s));
+ }
+
set_final_state_weights(rules);
return result;
}
@@ -106,7 +115,7 @@ namespace hfst
}
void ComposeIntersectLexicon::compute_state
- (HfstState state,ComposeIntersectRule * rules)
+ (HfstState state,ComposeIntersectRule * rules, bool allow_lexicon_epsilons)
{
StatePair p = get_pair(state);
@@ -120,42 +129,43 @@ namespace hfst
if (it->first == HfstTropicalTransducerTransitionData::get_number
("@_EPSILON_SYMBOL_@"))
{
- lexicon_skip_symbol_compose(it->second,p.second,state);
- lexicon_eps_transition_found = true;
+ if (allow_lexicon_epsilons)
+ {
+ lexicon_skip_symbol_compose(it->second,p.second,state);
+ lexicon_eps_transition_found = true;
+ }
}
- else if (is_flag_diacritic(it->first) and
- (not rules->known_symbol(it->first)))
+ else if (is_flag_diacritic(it->first) &&
+ (! rules->known_symbol(it->first)))
{
- lexicon_skip_symbol_compose(it->second,p.second,state);
+ lexicon_skip_symbol_compose(it->second,p.second,state);
lexicon_eps_transition_found = true;
- }
+ }
else
{
- compose(it->second,rules->get_transitions
- (p.second,it->first),state);
- }
+ compose(it->second,rules->get_transitions
+ (p.second,it->first),state);
+ }
}
- if (!lexicon_eps_transition_found)
- {
- rule_skip_symbol_compose
- (rules->get_transitions
- (p.second,HfstTropicalTransducerTransitionData::get_number
- ("@_EPSILON_SYMBOL_@")),p.first,state);
- }
+ rule_skip_symbol_compose
+ (rules->get_transitions
+ (p.second,HfstTropicalTransducerTransitionData::get_number
+ ("@_EPSILON_SYMBOL_@")),p.first,state);
}
-
+
+
void ComposeIntersectLexicon::lexicon_skip_symbol_compose
(const TransitionSet &transitions,HfstState rule_state,HfstState origin)
{
for (TransitionSet::const_iterator it = transitions.begin();
- it != transitions.end();
- ++it)
- {
- add_transition
- (origin,it->ilabel,it->olabel,it->weight,
- get_state(StatePair(it->target,rule_state)));
- }
+ it != transitions.end();
+ ++it)
+ {
+ add_transition
+ (origin,it->ilabel,it->olabel,it->weight,
+ get_state(StatePair(it->target,rule_state)));
+ }
}
void ComposeIntersectLexicon::rule_skip_symbol_compose
@@ -167,7 +177,7 @@ namespace hfst
{
add_transition
(origin,it->ilabel,it->olabel,it->weight,
- get_state(StatePair(lex_state,it->target)));
+ get_state(StatePair(lex_state,it->target), false));
}
}
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h
index 6201234..baab798 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectLexicon.h
@@ -2,6 +2,11 @@
#define COMPOSE_INTERSECT_LEXICON_H
#include <queue>
+#include <set>
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
#include "ComposeIntersectFst.h"
#include "ComposeIntersectRulePair.h"
@@ -20,22 +25,29 @@ namespace hfst
protected:
typedef std::pair<HfstState,HfstState> StatePair;
typedef std::map<StatePair,HfstState> StatePairMap;
+ typedef std::set<HfstState> StateSet;
+
typedef std::vector<StatePair> PairVector;
typedef std::queue<HfstState> StateQueue;
+
StatePairMap state_pair_map;
PairVector pair_vector;
StateQueue agenda;
HfstBasicTransducer result;
+ StateSet lexicon_non_epsilon_states;
bool is_flag_diacritic(size_t);
- HfstState get_state(const StatePair &);
+ HfstState get_state(const StatePair &, bool allow_lexicon_epsilons=true);
StatePair get_pair(HfstState);
void clear_all_info(void);
- HfstState map_state_and_add_to_agenda(const StatePair &);
+ HfstState map_state_and_add_to_agenda(const StatePair &,
+ bool allow_lexicon_epsilons);
HfstBasicTransducer &compute_composition_result
(ComposeIntersectRule *);
- void compute_state(HfstState state,ComposeIntersectRule *);
+ void compute_state(HfstState state,ComposeIntersectRule *,
+ bool allow_lexicon_epsilons);
+ bool can_have_lexicon_epsilons(HfstState s);
void set_final_state_weights(ComposeIntersectRule *);
void lexicon_skip_symbol_compose
(const TransitionSet &,HfstState,HfstState);
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc
index f673194..0f0c5b3 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.cc
@@ -6,7 +6,7 @@ namespace hfst
{
namespace implementations
{
- const HfstState ComposeIntersectRulePair::START;
+ const HfstState ComposeIntersectRulePair::START = 0;
ComposeIntersectRulePair::ComposeIntersectRulePair
(ComposeIntersectRule * fst1,ComposeIntersectRule * fst2):
@@ -34,9 +34,9 @@ namespace hfst
ComposeIntersectRulePair::get_transitions
(HfstState s,size_t symbol)
{
- if (not has_state(s))
+ if (! has_state(s))
{ HFST_THROW(StateNotDefined); }
- if (not transitions_computed(s,symbol))
+ if (! transitions_computed(s,symbol))
{ compute_transition_set(s,symbol); }
return state_transition_vector[s][symbol];
}
@@ -55,7 +55,7 @@ namespace hfst
HfstState ComposeIntersectRulePair::get_state(const StatePair &p)
{
- if (not has_pair(p))
+ if (! has_pair(p))
{
pair_state_map[p] = state_pair_vector.size();
state_pair_vector.push_back(p);
@@ -75,7 +75,7 @@ namespace hfst
float ComposeIntersectRulePair::get_final_weight(HfstState s) const
{
- if (not has_state(s))
+ if (! has_state(s))
{ HFST_THROW(StateNotDefined); }
const StatePair &state_pair = state_pair_vector[s];
return fst1->get_final_weight(state_pair.first) +
@@ -97,7 +97,7 @@ namespace hfst
(void)state_transition_vector[state][symbol];
TransitionSet transitions;
- while (it != fst1_transitions.end() and jt != fst2_transitions.end())
+ while (it != fst1_transitions.end() && jt != fst2_transitions.end())
{
if (it->olabel == jt->olabel)
{
@@ -150,7 +150,7 @@ std::ostream &ComposeIntersectRulePair::print(std::ostream &out)
}
out << s << "\t" << get_final_weight(s) << std::endl;
++s;
- if (not has_state(s))
+ if (! has_state(s))
{ break; }
}
return out;
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h
index 931415a..5d1a568 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectRulePair.h
@@ -18,7 +18,7 @@ namespace hfst
public:
typedef ComposeIntersectRule::TransitionSet TransitionSet;
- static const HfstState START = 0;
+ static const HfstState START; // = 0;
ComposeIntersectRulePair(ComposeIntersectRule *,ComposeIntersectRule *);
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc
index e01eaed..96e73fd 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.cc
@@ -25,7 +25,7 @@ int main(int argc, char * argv[])
IntSpaceSavingSet sset;
assert(sset.size() == 0);
- assert(not sset.has_element(0));
+ assert(! sset.has_element(0));
sset.insert(0);
assert(sset.size() == 1);
@@ -50,7 +50,7 @@ int main(int argc, char * argv[])
sset.clear();
assert(sset.size() == 0);
- assert(not sset.has_element(0));
+ assert(! sset.has_element(0));
sset.insert(0);
sset.insert(1);
assert(sset.has_element(1));
@@ -67,7 +67,7 @@ int main(int argc, char * argv[])
sset.clear();
assert(sset.size() == 0);
- assert(not sset.has_element(0));
+ assert(! sset.has_element(0));
sset.insert(1);
sset.insert(0);
assert(sset.has_element(0));
@@ -91,7 +91,7 @@ int main(int argc, char * argv[])
sset.clear();
sset.insert(2);
- assert(not sset.has_element(1));
+ assert(! sset.has_element(1));
std::cout << "ok" << std::endl;
return 0;
diff --git a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h
index 15927e1..09d43de 100644
--- a/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h
+++ b/libhfst/src/implementations/compose_intersect/ComposeIntersectUtilities.h
@@ -46,7 +46,7 @@ namespace hfst
{
iterator least_upper_bound = get_least_upper_bound(x);
const X &new_x = *least_upper_bound;
- if (least_upper_bound == end() or not (x == new_x))
+ if (least_upper_bound == end() || !(x == new_x))
{ add_value(x,least_upper_bound); }
}
@@ -88,7 +88,7 @@ namespace hfst
const_iterator it = container_.begin();
for ( ; it != container_.end(); ++it)
{
- if (not comparator(*it,x))
+ if (! comparator(*it,x))
{ break; }
}
return it;
@@ -99,7 +99,7 @@ namespace hfst
iterator it = container_.begin();
for ( ; it != container_.end(); ++it)
{
- if (not comparator(*it,x))
+ if (! comparator(*it,x))
{ break; }
}
return it;
diff --git a/libhfst/src/implementations/optimized-lookup/convert.cc b/libhfst/src/implementations/optimized-lookup/convert.cc
index a7f3363..18d19cc 100644
--- a/libhfst/src/implementations/optimized-lookup/convert.cc
+++ b/libhfst/src/implementations/optimized-lookup/convert.cc
@@ -539,7 +539,7 @@ TransitionTableIndex ConvertFstState::set_transition_table_indices(
++place;
// update the TransitionIndex's to store the table location of the
- associated transition
+ // associated transition
for(ConvertTransitionIndexSet::iterator it=transition_indices.begin();
it!=transition_indices.end(); ++it)
{
diff --git a/libhfst/src/implementations/optimized-lookup/pmatch.cc b/libhfst/src/implementations/optimized-lookup/pmatch.cc
index cd7383e..8518b2f 100644
--- a/libhfst/src/implementations/optimized-lookup/pmatch.cc
+++ b/libhfst/src/implementations/optimized-lookup/pmatch.cc
@@ -393,9 +393,9 @@ std::string PmatchContainer::parse_name_from_hfst3_header(std::istream & f)
if (headervalue[remaining_header_len - 1] != '\0') {
HFST_THROW(TransducerHeaderException);
}
- char type[remaining_header_len];
+ char * type = new char [remaining_header_len];
bool type_defined = false;
- char name[remaining_header_len];
+ char * name = new char [remaining_header_len];
bool name_defined = false;
int i = 0;
while (i < remaining_header_len) {
@@ -421,7 +421,10 @@ std::string PmatchContainer::parse_name_from_hfst3_header(std::istream & f)
if (strcmp(type, "HFST_OLW") != 0) {
HFST_THROW(TransducerHeaderException);
}
- return std::string(name);
+ std::string retval = std::string(name);
+ delete [] type;
+ delete [] name;
+ return retval;
} else // nope. put back what we've taken
{
f.unget(); // first the non-matching character
@@ -481,6 +484,7 @@ void PmatchContainer::process(std::string & input_str)
++line_number;
output.clear();
locations.clear();
+ DoubleTape nonmatching_locations;
while (has_queued_input(input_pos)) {
SymbolNumber current_input = input[input_pos];
if (not_possible_first_symbol(current_input)) {
@@ -488,15 +492,27 @@ void PmatchContainer::process(std::string & input_str)
++input_pos;
if (locate_mode && alphabet.is_printable(current_input)) {
++printable_input_pos;
+ nonmatching_locations.push_back(
+ SymbolPair(current_input, current_input));
}
continue;
}
tape.clear();
unsigned int tape_pos = 0;
+ unsigned int old_input_pos = input_pos;
toplevel->match(input_pos, tape_pos);
if (tape_pos > 0) {
// Tape moved
if (locate_mode) {
+ if (!nonmatching_locations.empty()) {
+ LocationVector ls;
+ Location nonmatching = alphabet.locatefy(printable_input_pos - nonmatching_locations.size(),
+ WeightedDoubleTape(nonmatching_locations, 0.0));
+ nonmatching.output = "@_NONMATCHING_@";
+ ls.push_back(nonmatching);
+ locations.push_back(ls);
+ nonmatching_locations.clear();
+ }
LocationVector ls;
for (WeightedDoubleTapeVector::iterator it = (toplevel->locations)->begin();
it != (toplevel->locations)->end(); ++it) {
@@ -505,17 +521,29 @@ void PmatchContainer::process(std::string & input_str)
}
sort(ls.begin(), ls.end());
locations.push_back(ls);
+ printable_input_pos += (input_pos - old_input_pos);
} else {
copy_to_output(toplevel->get_best_result());
}
- } else {
+ }
+ if (tape_pos == 0 || input_pos == old_input_pos) {
+ // If nothing happened, we move one position up
copy_to_output(current_input, current_input);
++input_pos;
if (locate_mode && alphabet.is_printable(current_input)) {
++printable_input_pos;
+ nonmatching_locations.push_back(SymbolPair(current_input, current_input));
}
}
}
+ if (locate_mode && !nonmatching_locations.empty()) {
+ LocationVector ls;
+ Location nonmatching = alphabet.locatefy(printable_input_pos - nonmatching_locations.size(),
+ WeightedDoubleTape(nonmatching_locations, 0.0));
+ nonmatching.output = "@_NONMATCHING_@";
+ ls.push_back(nonmatching);
+ locations.push_back(ls);
+ }
}
std::string PmatchContainer::match(std::string & input)
@@ -943,7 +971,7 @@ void PmatchContainer::initialize_input(const char * input_s)
// even if it's not utf-8, tokenize a byte at a time
bytes_to_tokenize = 1;
}
- char new_symbol[bytes_to_tokenize + 1];
+ char * new_symbol = new char [bytes_to_tokenize + 1];
memcpy(new_symbol, *input_str_ptr, bytes_to_tokenize);
new_symbol[bytes_to_tokenize] = '\0';
(*input_str_ptr) += bytes_to_tokenize;
@@ -951,6 +979,7 @@ void PmatchContainer::initialize_input(const char * input_s)
encoder->read_input_symbol(new_symbol, symbol_count);
k = symbol_count;
++symbol_count;
+ delete [] new_symbol;
}
input.push_back(k);
}
@@ -1231,13 +1260,18 @@ void PmatchTransducer::take_transitions(SymbolNumber input,
// input tape to find the symbol we want to write
this_output = container->input[input_pos];
}
+ if (this_input == alphabet.get_identity_symbol() ||
+ (this_input == alphabet.get_unknown_symbol()) ||
+ (alphabet.list2symbols[this_input] != NO_SYMBOL_NUMBER)) {
+ this_input = container->input[input_pos];
+ }
Weight tmp = local_stack.top().running_weight;
local_stack.top().running_weight +=
transition_table[i].get_weight();
if (this_input == alphabet.get_special(Pmatch_passthrough)) {
get_analyses(input_pos, tape_pos, target); // FIXME
} else {
- container->tape.write(tape_pos, input, this_output);
+ container->tape.write(tape_pos, this_input, this_output);
get_analyses(input_pos + 1, tape_pos + 1, target);
}
local_stack.top().running_weight = tmp;
diff --git a/libhfst/src/implementations/optimized-lookup/pmatch.h b/libhfst/src/implementations/optimized-lookup/pmatch.h
index b97f2dd..e6ec1f3 100644
--- a/libhfst/src/implementations/optimized-lookup/pmatch.h
+++ b/libhfst/src/implementations/optimized-lookup/pmatch.h
@@ -190,10 +190,8 @@ namespace hfst_ol {
// LocationVector locatefy_output(void);
static std::string parse_name_from_hfst3_header(std::istream & f);
void set_verbose(bool b) { verbose = b; }
- void set_locate_mode(bool b) {
- locate_mode = b;
- alphabet.extract_tags = b;
- }
+ void set_extract_tags_mode(bool b)
+ { alphabet.extract_tags = b; }
void set_profile(bool b) { profile_mode = b; }
bool try_recurse(void)
{
diff --git a/libhfst/src/implementations/optimized-lookup/transducer.cc b/libhfst/src/implementations/optimized-lookup/transducer.cc
index 3bdcf7d..b722e58 100644
--- a/libhfst/src/implementations/optimized-lookup/transducer.cc
+++ b/libhfst/src/implementations/optimized-lookup/transducer.cc
@@ -267,13 +267,14 @@ bool Transducer::initialize_input(const char * input)
if (bytes_to_tokenize == 0) {
return false; // tokenization failed
}
- char new_symbol[bytes_to_tokenize + 1];
+ char * new_symbol = new char[bytes_to_tokenize + 1];
memcpy(new_symbol, *input_str_ptr, bytes_to_tokenize);
new_symbol[bytes_to_tokenize] = '\0';
(*input_str_ptr) += bytes_to_tokenize;
alphabet->add_symbol(new_symbol);
k = alphabet->get_symbol_table().size() - 1;
encoder->read_input_symbol(new_symbol, k);
+ delete [] new_symbol;
}
input_tape.write(i, k);
++i;
diff --git a/libhfst/src/implementations/optimized-lookup/transducer.h b/libhfst/src/implementations/optimized-lookup/transducer.h
index 4e6990b..b86ecb8 100644
--- a/libhfst/src/implementations/optimized-lookup/transducer.h
+++ b/libhfst/src/implementations/optimized-lookup/transducer.h
@@ -14,6 +14,12 @@
#ifndef _HFST_OL_TRANSDUCER_TRANSDUCER_H_
#define _HFST_OL_TRANSDUCER_TRANSDUCER_H_
+#ifndef _MSC_VER
+# include <unistd.h>
+#else
+# include <io.h>
+#endif
+
#include <vector>
#include <set>
#include <iostream>
@@ -31,6 +37,12 @@
#include "../../HfstFlagDiacritics.h"
#include "../../HfstSymbolDefs.h"
+#ifdef _MSC_VER
+ #include <BaseTsd.h>
+ typedef SSIZE_T ssize_t;
+#endif
+
+
/** \brief A namespace for optimized-lookup functions and datatypes.*/
namespace hfst_ol {
using hfst::FdOperation;
@@ -75,10 +87,11 @@ struct TraversalState
};
typedef std::set<TraversalState> TraversalStates;
-const SymbolNumber NO_SYMBOL_NUMBER = std::numeric_limits<SymbolNumber>::max();
+ // parentheses avoid collision with windows macro 'max'
+ const SymbolNumber NO_SYMBOL_NUMBER = (std::numeric_limits<SymbolNumber>::max)();
const TransitionTableIndex NO_TABLE_INDEX =
- std::numeric_limits<TransitionTableIndex>::max();
-const unsigned long NO_COUNTER = std::numeric_limits<unsigned long>::max();
+ (std::numeric_limits<TransitionTableIndex>::max)();
+ const unsigned long NO_COUNTER = (std::numeric_limits<unsigned long>::max)();
const Weight INFINITE_WEIGHT = static_cast<float>(NO_TABLE_INDEX);
enum HeaderFlag {Weighted, Deterministic, Input_deterministic, Minimized,
@@ -458,8 +471,8 @@ public:
void write(std::ostream& os, bool weighted) const
{
os.write(reinterpret_cast<const char*>(&input_symbol),
- sizeof(input_symbol));
- if(!weighted and input_symbol == NO_SYMBOL_NUMBER and
+ sizeof(SymbolNumber));
+ if(!weighted && input_symbol == NO_SYMBOL_NUMBER &&
first_transition_index != NO_TABLE_INDEX) {
// Make sure that we write the correct type of final index
unsigned int unweighted_final_index = 1;
diff --git a/libhfst/src/parsers/LexcCompiler.cc b/libhfst/src/parsers/LexcCompiler.cc
index 67cd416..9199ffb 100644
--- a/libhfst/src/parsers/LexcCompiler.cc
+++ b/libhfst/src/parsers/LexcCompiler.cc
@@ -41,7 +41,8 @@ using std::set_difference;
#include "lexc-utils.h"
#include "lexc-parser.hh"
#include "xre_utils.h"
-#include "../../../tools/src/HfstStrings2FstTokenizer.h"
+#include "HfstSymbolDefs.h"
+//#include "../../../tools/src/HfstStrings2FstTokenizer.h"
using hfst::HfstTransducer;
using hfst::implementations::HfstTransitionGraph;
@@ -50,7 +51,7 @@ using hfst::implementations::HfstState;
using hfst::implementations::HfstBasicTransition;
using hfst::ImplementationType;
using hfst::xre::XreCompiler;
-
+using hfst::StringVector;
//using same alg as strings to fst
static char *epsilonname=NULL; // FIX: use this
@@ -65,12 +66,14 @@ static StringVector multichar_symbols;
extern FILE* hlexcin;
extern int hlexcparse();
extern int hlexcnerrs;
-extern void hlexclex_destroy();
+extern int hlexclex_destroy();
#ifndef DEBUG_MAIN
namespace hfst { namespace lexc {
+ bool debug = false;
+
LexcCompiler* lexc_ = 0;
LexcCompiler::LexcCompiler() :
@@ -871,17 +874,23 @@ LexcCompiler::compileLexical()
joinersAll.repeat_star();
joinersAll.minimize();
-/*
- printf("lexicons before compose: \n");
- lexicons.write_in_att_format(stdout, 1);
-
- printf("joinersAll: \n");
- joinersAll.write_in_att_format(stdout, 1);
- printf("\n");
-*/
+ if (debug)
+ {
+ fprintf(stderr, "lexicons before compose: \n");
+ lexicons.write_in_att_format(stderr, 1);
+
+ fprintf(stderr, "joinersAll: \n");
+ joinersAll.write_in_att_format(stderr, 1);
+ fprintf(stderr, "\n");
+ }
lexicons.compose(joinersAll).minimize();
+ if (debug)
+ {
+ std::cerr << "lexicons after composition: " << std::endl;
+ std::cerr << lexicons << std::endl;
+ }
HfstSymbolSubstitutions allSubstitutions;
if(with_flags_)
@@ -899,7 +908,10 @@ LexcCompiler::compileLexical()
s != transducerAlphabet.end();
++s)
{
- //printf("%s \n", s->c_str());
+ if (debug)
+ {
+ fprintf(stderr, "handling alpha: '%s'...\n", s->c_str());
+ }
String alph = *s;
if ( alph[0] == '$' && *alph.rbegin() == '$' && alph.size() > 2)
@@ -907,6 +919,10 @@ LexcCompiler::compileLexical()
replace(alph.begin(), alph.end(), '$', '@');
//std::cout << alph << '\n';
fakeFlagsToRealFlags.insert(StringPair(*s, alph));
+ if (debug)
+ {
+ std::cerr << "debug: inserting fakeFlagsToRealFlags replacement: " << *s << " -> " << alph << std::endl;
+ }
//lexicons.substitute(*s, alph).minimize();
}
}
@@ -925,8 +941,12 @@ LexcCompiler::compileLexical()
lexicons.substitute(allSubstitutions).minimize();
lexicons.prune_alphabet();
-
-
+ if (debug)
+ {
+ std::cerr << "lexicons after substitution: " << std::endl;
+ std::cerr << lexicons << std::endl;
+ }
+
//replace reg exp key with transducers
if (verbose_)
{
@@ -948,6 +968,10 @@ LexcCompiler::compileLexical()
//std::cout << alph << '\n';
fakeRegexprToReal.insert(StringPair(it->first, alph));
+ if (debug)
+ {
+ std::cerr << "debug: inserting fakeRegexprToReal replacement: " << it->first << " -> " << alph << std::endl;
+ }
// lexicons.substitute(it->first, alph).minimize();
}
@@ -979,16 +1003,25 @@ LexcCompiler::compileLexical()
}
HfstBasicTransducer btr(*(it->second));
regMarkToTr[alph] = btr;
+ if (debug)
+ {
+ std::cerr << "debug: regMarkToTr[" << alph << "] = " << std::endl;
+ btr.write_in_att_format(stderr);
+ }
//lexicons.substitute(StringPair(alph, alph), *it->second, true).minimize();
}
-
HfstBasicTransducer lexicons_basic(lexicons);
lexicons_basic.substitute(regMarkToTr, true);
+ lexicons_basic.prune_alphabet();
+ if (debug)
+ {
+ std::cerr << "lexicons_basic after regexp substitution: " << std::endl;
+ lexicons_basic.write_in_att_format(stderr);
+ }
- lexicons_basic.prune_alphabet();
/*
printf("lexicons after substitute: \n");
@@ -1002,10 +1035,9 @@ LexcCompiler::compileLexical()
// Preserve only first flag of consecutive P and R lexname flag series,
// e.g. change P.LEXNAME.1 R.LEXNAME.1 P.LEXNAME.2 R.LEXNAME.2 into P.LEXNAME.1
-
+
if (with_flags_ && minimize_flags_)
{
-
// To substitute "@[P|R].LEXNAME...@" with "$[P|R].LEXNAME...$" and vice versa.
std::map<String, String> flag_substitutions; // @ -> $
std::map<String, String> reverse_flag_substitutions; // $ -> @
@@ -1064,8 +1096,10 @@ LexcCompiler::compileLexical()
std::string context_regexp(flag_remover_regexp);
flag_remover_regexp.append(" -> 0 || ").append(context_regexp).append(" _ ");
- // DEBUG
- //fprintf(stderr, "flag_remover_regexp: %s\n", flag_remover_regexp.c_str());
+ if (debug)
+ {
+ fprintf(stderr, "flag_remover_regexp: %s\n", flag_remover_regexp.c_str());
+ }
hfst::xre::XreCompiler xre_comp(format_);
HfstTransducer * flag_filter = xre_comp.compile(flag_remover_regexp);
@@ -1106,7 +1140,7 @@ LexcCompiler::compileLexical()
}
else
{
- // Convert symbols "$[P|R].LEXNAME...$" back to "@[P|R].LEXNAME...@"
+ // Convert symbols "$[P|R].LEXNAME...$" back to "@[P|R].LEXNAME...@"
filtered_lexicons.substitute(reverse_flag_substitutions);
}
diff --git a/libhfst/src/parsers/pmatch_lex.ll b/libhfst/src/parsers/pmatch_lex.ll
index e13e3d7..5cb758e 100644
--- a/libhfst/src/parsers/pmatch_lex.ll
+++ b/libhfst/src/parsers/pmatch_lex.ll
@@ -17,7 +17,7 @@
#define YY_INPUT(buf, retval, maxlen) (retval = hfst::pmatch::getinput(buf, maxlen))
extern
-void pmatcherror(char *text);
+int pmatcherror(char *text);
%}
@@ -49,8 +49,9 @@ WEIGHT [0-9]+(\.[0-9]+)?
/* token character */
NAME_CH {A7UNRESTRICTED}|{U8H}|{EC}
-UINTEGER [1-9][0-9]*
-INTEGER -?[1-9][0-9]*
+ZERO "0"
+UINTEGER ([1-9][0-9]*)|{ZERO}
+INTEGER (-?[1-9][0-9]*)|{ZERO}
WSP [\t ]
LWSP [\t\r\n ]
@@ -259,10 +260,11 @@ return REGEX;
return CATENATE_N;
}
-".r" { return REVERSE; }
-".i" { return INVERT; }
-".u" { return UPPER; }
-".l" { return LOWER; }
+".r" { return REVERSE; }
+".i" { return INVERT; }
+".u" { return PMATCH_UPPER; }
+".l" { return PMATCH_LOWER; }
+".t(" { return TAG_LEFT; }
"@bin\""[^""]+"\""|"@\""[^""]+"\"" {
pmatchlval.label = hfst::pmatch::get_escaped_delimited(pmatchtext, '"');
diff --git a/libhfst/src/parsers/pmatch_parse.yy b/libhfst/src/parsers/pmatch_parse.yy
index b64c70c..499c626 100644
--- a/libhfst/src/parsers/pmatch_parse.yy
+++ b/libhfst/src/parsers/pmatch_parse.yy
@@ -19,7 +19,7 @@
#include "pmatch_utils.h"
using hfst::pmatch::PmatchAstNode;
- extern void pmatcherror(const char * text);
+ extern int pmatcherror(const char * text);
extern int pmatchlex();
extern int pmatchlineno;
@@ -130,7 +130,7 @@ LOWER_PRIORITY_UNION
%nonassoc SUBSTITUTE_LEFT TERM_COMPLEMENT
%nonassoc COMPLEMENT CONTAINMENT CONTAINMENT_ONCE CONTAINMENT_OPT
-%nonassoc REVERSE INVERT UPPER LOWER STAR PLUS
+%nonassoc REVERSE INVERT PMATCH_UPPER PMATCH_LOWER STAR PLUS
%nonassoc <values> CATENATE_N_TO_K
%nonassoc <value> CATENATE_N CATENATE_N_PLUS CATENATE_N_MINUS
@@ -146,7 +146,7 @@ PAIR_SEPARATOR_WO_RIGHT PAIR_SEPARATOR_WO_LEFT
%nonassoc DEFINE REGEX DEFINED_LIST DEFINS DEFFUN ALPHA LOWERALPHA UPPERALPHA
NUM PUNCT WHITESPACE OPTCAP_LEFT TOLOWER_LEFT TOUPPER_LEFT INS_LEFT DEFINE_LEFT
ENDTAG_LEFT LC_LEFT RC_LEFT NLC_LEFT NRC_LEFT MAP_LEFT LIT_LEFT LST_LEFT
-SIGMA_LEFT COUNTER_LEFT OR_LEFT AND_LEFT
+SIGMA_LEFT COUNTER_LEFT OR_LEFT AND_LEFT TAG_LEFT
%%
@@ -404,10 +404,10 @@ FUNCBODY5: FUNCBODY6 { }
| FUNCBODY5 INVERT {
$$ = new PmatchAstNode($1, hfst::pmatch::AstInvert);
}
-| FUNCBODY5 UPPER {
+| FUNCBODY5 PMATCH_UPPER {
$$ = new PmatchAstNode($1, hfst::pmatch::AstInputProject);
}
-| FUNCBODY5 LOWER {
+| FUNCBODY5 PMATCH_LOWER {
$$ = new PmatchAstNode($1, hfst::pmatch::AstOutputProject);
}
| FUNCBODY5 CATENATE_N {
@@ -708,23 +708,24 @@ REPLACE : REGEXP3 {}
}
;
-PARALLEL_RULES: RULE
+PARALLEL_RULES:
+PARALLEL_RULES COMMACOMMA RULE
{
- // std::cerr << "parallel_rules:rule"<< std::endl;
+// std::cerr << "parallel_rules: parallel_rules ,, rule"<< std::endl;
+ Rule tmpRule($3->second);
+ $1->second.push_back(tmpRule);
+ $$ = new std::pair< ReplaceArrow, std::vector<Rule> > ($3->first, $1->second);
+ delete $3;
+}
+| RULE {
+// std::cerr << "parallel_rules:rule"<< std::endl;
std::vector<Rule> * ruleVector = new std::vector<Rule>();
ruleVector->push_back($1->second);
$$ = new std::pair< ReplaceArrow, std::vector<Rule> > ($1->first, *ruleVector);
delete $1;
}
-| PARALLEL_RULES COMMACOMMA RULE
-{
- // std::cerr << "parallel_rules: parallel_rules ,, rule"<< std::endl;
- Rule tmpRule($3->second);
- $1->second.push_back(tmpRule);
- $$ = new std::pair< ReplaceArrow, std::vector<Rule> > ($3->first, $1->second);
- delete $3;
-}
+
;
RULE: MAPPINGPAIR_VECTOR
@@ -1040,8 +1041,7 @@ REGEXP6: REGEXP7 { }
REGEXP7: REGEXP8 { }
| REGEXP7 IGNORING REGEXP8 {
- pmatcherror("No ignoring");
- $$ = $1;
+ $$ = & $1->insert_freely(*$3, false);
delete $3;
}
| REGEXP7 IGNORE_INTERNALLY REGEXP8 {
@@ -1122,10 +1122,10 @@ REGEXP9: REGEXP10 { }
| REGEXP9 INVERT {
$$ = & $1->invert();
}
-| REGEXP9 UPPER {
+| REGEXP9 PMATCH_UPPER {
$$ = & $1->input_project();
}
-| REGEXP9 LOWER {
+| REGEXP9 PMATCH_LOWER {
$$ = & $1->output_project();
}
| REGEXP9 CATENATE_N {
@@ -1164,11 +1164,20 @@ REGEXP11: REGEXP12 { }
| LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
$$ = $2;
}
+| LEFT_BRACKET REGEXP2 RIGHT_BRACKET TAG_LEFT SYMBOL RIGHT_PARENTHESIS {
+ hfst::pmatch::add_end_tag($2, $5);
+ $$ = hfst::pmatch::add_pmatch_delimiters($2);
+}
+| LEFT_BRACKET REGEXP2 RIGHT_BRACKET TAG_LEFT QUOTED_LITERAL RIGHT_PARENTHESIS {
+ hfst::pmatch::add_end_tag($2, $5);
+ $$ = hfst::pmatch::add_pmatch_delimiters($2);
+}
| LEFT_PARENTHESIS REGEXP2 RIGHT_PARENTHESIS {
$$ = & $2->optionalize();
}
| ALPHA {
$$ = new HfstTransducer(*hfst::pmatch::get_utils()->latin1_alpha_acceptor);
+ $$->minimize();
}
| LOWERALPHA {
$$ = new HfstTransducer(*hfst::pmatch::get_utils()->latin1_lowercase_acceptor);
@@ -1274,6 +1283,7 @@ LABEL_PAIR: LABEL PAIR_SEPARATOR LABEL {
}
| ANY_TOKEN PAIR_SEPARATOR LABEL {
$$ = new HfstTransducer(hfst::internal_unknown, $3, hfst::pmatch::format);
+ $$->disjunct(HfstTransducer($3, $3, hfst::pmatch::format));
free($3);
}
| LABEL PAIR_SEPARATOR_WO_RIGHT {
@@ -1286,6 +1296,7 @@ LABEL_PAIR: LABEL PAIR_SEPARATOR LABEL {
}
| PAIR_SEPARATOR_WO_LEFT LABEL {
$$ = new HfstTransducer(hfst::internal_unknown, $2, hfst::pmatch::format);
+ $$->disjunct(HfstTransducer($2, $2, hfst::pmatch::format));
free($2);
}
| PAIR_SEPARATOR_WO_LEFT ANY_TOKEN {
diff --git a/libhfst/src/parsers/pmatch_utils.cc b/libhfst/src/parsers/pmatch_utils.cc
index 4a33c63..b625a29 100644
--- a/libhfst/src/parsers/pmatch_utils.cc
+++ b/libhfst/src/parsers/pmatch_utils.cc
@@ -11,7 +11,7 @@
#include "pmatch_utils.h"
#include "HfstTransducer.h"
-#include "tools/src/HfstUtf8.h"
+//#include "tools/src/HfstUtf8.h"
#include "implementations/optimized-lookup/pmatch.h"
using std::string;
@@ -696,30 +696,35 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
std::cerr << "\nHarmonizing... ";
}
- HfstTransducer dummy(format);
- // We keep TOP and any inserted transducers
- std::map<std::string, hfst::HfstTransducer *>::iterator defs_itr;
- for (defs_itr = definitions.begin(); defs_itr != definitions.end();
- ++defs_itr) {
- if (defs_itr->first.compare("TOP") == 0 ||
- inserted_transducers.count(defs_itr->first) != 0) {
- dummy.harmonize(*defs_itr->second);
+ if (inserted_transducers.size() > 0) {
+
+ HfstTransducer dummy(format);
+ // We keep TOP and any inserted transducers
+ std::map<std::string, hfst::HfstTransducer *>::iterator defs_itr;
+ for (defs_itr = definitions.begin(); defs_itr != definitions.end();
+ ++defs_itr) {
+ if (defs_itr->first.compare("TOP") == 0 ||
+ inserted_transducers.count(defs_itr->first) != 0) {
+ dummy.harmonize(*defs_itr->second);
+ }
}
- }
-
- // Now that dummy is harmonized with everything, we harmonize everything
- // with dummy and insert them into the result
- for(defs_itr = definitions.begin(); defs_itr != definitions.end();
- ++defs_itr) {
- if (defs_itr->first.compare("TOP") == 0 ||
- inserted_transducers.count(defs_itr->first) != 0) {
- dummy.harmonize(*defs_itr->second);
- retval.insert(std::pair<std::string, hfst::HfstTransducer*>(
- defs_itr->first,
- defs_itr->second));
- } else {
- delete defs_itr->second;
+
+ // Now that dummy is harmonized with everything, we harmonize everything
+ // with dummy and insert them into the result
+ for(defs_itr = definitions.begin(); defs_itr != definitions.end();
+ ++defs_itr) {
+ if (defs_itr->first.compare("TOP") == 0 ||
+ inserted_transducers.count(defs_itr->first) != 0) {
+ dummy.harmonize(*defs_itr->second);
+ retval.insert(std::pair<std::string, hfst::HfstTransducer*>(
+ defs_itr->first,
+ defs_itr->second));
+ } else {
+ delete defs_itr->second;
+ }
}
+ } else {
+ retval.insert(std::pair<std::string, hfst::HfstTransducer*>("TOP", definitions["TOP"]));
}
if (hfst::pmatch::verbose) {
double duration = (clock() - hfst::pmatch::timer) /
diff --git a/libhfst/src/parsers/pmatch_utils.h b/libhfst/src/parsers/pmatch_utils.h
index 817a76b..a43cb4c 100644
--- a/libhfst/src/parsers/pmatch_utils.h
+++ b/libhfst/src/parsers/pmatch_utils.h
@@ -181,13 +181,24 @@ template<typename T, size_t N>
"ß"
};
+ // MSVC compiler complains about \u strings...
+
static const char * combining_accents[] =
{
// Combining accents: grave, acute, circumflex, tilde, overline,
// diaresis, charon, cedilla
+#ifndef _MSC_VER
"\u0300", "\u0301", "\u0302", "\u0303", "\u0305", "\u0308", "\u030C", "\u0327",
+#else
+ "\xCC\x80", "\xCC\x81", "\xCC\x82", "\xCC\x83", "\xCC\x85", "\xCC\x88", "\xCC\x8C", "\xCC\xA7",
+#endif
+
// Small solidus and large combining solidus
+#ifndef _MSC_VER
"\u0337", "\u0338"
+#else
+ "\xCC\xB7", "\xCC\xB8"
+#endif
};
static const char * latin1_punct[] =
@@ -201,10 +212,18 @@ template<typename T, size_t N>
{
" ", "\n", "\t",
// Non-breaking space, CR
+#ifndef _MSC_VER
"\u00A0",
+#else
+ "\xC2\xA0",
+#endif
"\r",
// punctuation space, thin space, line separator, par separator
+#ifndef _MSC_VER
"\u2008", "\u2009", "\u2028", "\u2029"
+#else
+ "\xE2\x80\x88", "\xE2\x80\x89", "\xE2\x80\xA8", "\xE2\x80\xA9"
+#endif
};
diff --git a/libhfst/src/parsers/xre_lex.ll b/libhfst/src/parsers/xre_lex.ll
index 79f13cc..666302f 100644
--- a/libhfst/src/parsers/xre_lex.ll
+++ b/libhfst/src/parsers/xre_lex.ll
@@ -1,3 +1,4 @@
+
%option 8Bit batch noyylineno noyywrap reentrant bison-bridge prefix="xre"
%{
@@ -168,8 +169,8 @@ BRACED [{]([^}]|[\300-\337].|[\340-\357]..|[\360-\367]...)+[}]
".r" { CR; return REVERSE; }
".i" { CR; return INVERT; }
-".u" { CR; return UPPER; }
-".l" { CR; return LOWER; }
+".u" { CR; return XRE_UPPER; }
+".l" { CR; return XRE_LOWER; }
"@bin\""[^""]+"\""|"@\""[^""]+"\"" {
CR;
diff --git a/libhfst/src/parsers/xre_parse.yy b/libhfst/src/parsers/xre_parse.yy
index 2593c0b..553a59e 100644
--- a/libhfst/src/parsers/xre_parse.yy
+++ b/libhfst/src/parsers/xre_parse.yy
@@ -47,7 +47,7 @@ using hfst::xre::harmonize_;
using hfst::xre::harmonize_flags_;
union YYSTYPE;
-class yy_buffer_state;
+struct yy_buffer_state;
typedef yy_buffer_state * YY_BUFFER_STATE;
typedef void * yyscan_t;
@@ -147,7 +147,7 @@ int xrelex ( YYSTYPE * , yyscan_t );
%nonassoc SUBSTITUTE_LEFT TERM_COMPLEMENT
%nonassoc COMPLEMENT CONTAINMENT CONTAINMENT_ONCE CONTAINMENT_OPT
-%nonassoc REVERSE INVERT UPPER LOWER STAR PLUS
+%nonassoc REVERSE INVERT XRE_UPPER XRE_LOWER STAR PLUS
%nonassoc <values> CATENATE_N_TO_K
%nonassoc <value> CATENATE_N CATENATE_N_PLUS CATENATE_N_MINUS
@@ -315,6 +315,18 @@ REGEXP2: REPLACE
HfstTransducer replaceTr(hfst::xre::format);
replaceTr = replace(rule, false);
+ // if we are replacing with flag diacritics, the rule must allow
+ // flags to be replaced with themselves
+ StringSet alpha = $3->get_alphabet();
+ for (StringSet::const_iterator it = alpha.begin(); it != alpha.end(); it++)
+ {
+ if (FdOperation::is_diacritic(*it))
+ {
+ replaceTr.insert_freely(StringPair(*it, *it), false);
+ }
+ }
+ replaceTr.minimize();
+
tmpTr->compose(replaceTr).minimize();
tmpTr->invert().compose(replaceTr).invert();
}
@@ -866,10 +878,10 @@ REGEXP9: REGEXP10 { }
| REGEXP9 INVERT {
$$ = & $1->invert();
}
- | REGEXP9 UPPER {
+ | REGEXP9 XRE_UPPER {
$$ = & $1->input_project();
}
- | REGEXP9 LOWER {
+ | REGEXP9 XRE_LOWER {
$$ = & $1->output_project();
}
| REGEXP9 CATENATE_N {
diff --git a/libhfst/src/parsers/xre_utils.cc b/libhfst/src/parsers/xre_utils.cc
index 996bfe3..ce9db5b 100644
--- a/libhfst/src/parsers/xre_utils.cc
+++ b/libhfst/src/parsers/xre_utils.cc
@@ -16,7 +16,7 @@ using std::string;
using std::map;
using std::ostringstream;
-class yy_buffer_state;
+struct yy_buffer_state;
typedef yy_buffer_state * YY_BUFFER_STATE;
typedef void * yyscan_t;
extern int xreparse(yyscan_t);
@@ -50,7 +50,7 @@ int xreerror(yyscan_t scanner, const char* msg)
hfst::xre::data, xreget_text(scanner));
}
- buffer[1023] == '\0';
+ buffer[1023] = '\0';
hfst::xre::error_message = std::string(buffer);
return 0;
}
@@ -60,7 +60,7 @@ xreerror(const char *msg)
{
char buffer [1024];
int n = sprintf(buffer, "*** xre parsing failed: %s\n", msg);
- buffer[1023] == '\0';
+ buffer[1023] = '\0';
hfst::xre::error_message = std::string(buffer);
return 0;
}
diff --git a/scripts/README_eight_tools.txt b/scripts/README_eight_tools.txt
new file mode 100644
index 0000000..b3612af
--- /dev/null
+++ b/scripts/README_eight_tools.txt
@@ -0,0 +1,91 @@
+
+This package contains the following command line tools for 64-bit Windows:
+
+ - hfst-xfst
+ - hfst-lexc
+ - hfst-twolc.bat
+ - hfst-lookup
+ - hfst-optimized-lookup
+ - hfst-proc
+ - hfst-pmatch
+ - hfst-pmatch2fst
+
+
+For more info about the tools, see their KitWiki pages:
+
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstLexc>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstTwolC>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstLookUp>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstOptimizedLookup>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstProc>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPmatch>
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstPmatch2Fst>
+
+or run hfst-TOOLNAME --help in command prompt.
+
+
+For more info about the HFST project, see:
+ <http://hfst.sourceforge.net/>
+
+
+Usage
+-----
+
+The tools hfst-xfst, hfst-lookup, hfst-optimized-lookup and hfst-pmatch work
+by default in an interactive mode. They can be started either by
+double-clicking the icon (opens up a new console window) or executing
+'hfst-[xfst|lookup|optimized-lookup|pmatch]' in command prompt (runs the
+program in the same window). They read input from the console, i.e. line by
+line from the user and print their output to the console. Both control+D and
+control+Z can be used to signal end of user input and exit the program.
+
+The tools hfst-lexc, hfst-twolc.bat and hfst-pmatch2fst are meant to be used
+from the command prompt. They take their input from a file and write their
+input to a file. The files can be given as a command line arguments or via
+standard streams. hfst-proc has currently no support for UTF-8 in interactive
+mode on windows, so it should also be used in the same way.
+
+Note that hfst-twolc.bat is a script that uses programs htwolcpre1, htwolcpre2
+and htwolcpre3, so they should all be located in the same directory.
+
+
+Character encodings
+-------------------
+
+All HFST tools work in a UTF-8 mode. When they are used in interactive mode,
+they set the console code point to 65001, i.e. UTF-8. Input text files must be
+encoded as UTF-8 with no BOM (byte order mark) included.
+
+
+Input and output via standard streams
+-------------------------------------
+
+As windows separates between console input and standard input as well as
+console output and standard output, some of the tools have an option
+--pipe-mode. It specifies if input is redirected from a file or output
+redirected to a file or both (the default for this option). --pipe-mode is
+relevant only for text streams, binary streams work as such. It is always
+possible to specify input and output text files also as command line arguments.
+
+
+hfst-xfst
+---------
+
+A prompt 'hfst[N]:' is shown when the program runs, N being the number of
+transducers in the stack at the moment. All read and write commands happen in
+the directory where the program was started.
+
+When an interactive command is run inside hfst-xfst (e.g. 'apply up'), both
+control+D and control+Z can be used to exit that command.
+
+The command 'exit' exits the program and closes the console (if a new one was
+opened) or returns to the console where the program was started.
+
+When run from the command line, a number of options are available. For more
+info, see the page <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>.
+Two important options related to input and output processing are:
+
+ -F, --scriptfile=FILE Read commands from FILE, and quit
+ -l, --startupfile=FILE Read commands from FILE on startup
+
diff --git a/scripts/README_xfst.txt b/scripts/README_xfst.txt
new file mode 100644
index 0000000..5e1fb46
--- /dev/null
+++ b/scripts/README_xfst.txt
@@ -0,0 +1,59 @@
+
+This package contains the command line tool hfst-xfst.exe for 64-bit Windows.
+
+For more info about the tool, see:
+ <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>
+
+For more info about the HFST project, see:
+ <http://hfst.sourceforge.net/>
+
+
+Usage
+-----
+
+The tool works by default in an interactive mode. It can be started either by
+double-clicking the icon (opens up a new console window) or executing
+'hfst-xfst' in command prompt (runs the program in the same window).
+
+A prompt 'hfst[N]:' is shown when the program runs, N being the number of
+transducers in the stack at the moment. All read and write commands happen in
+the directory where the program was started.
+
+When an interactive command is run inside hfst-xfst (e.g. 'apply up'), both
+control+D and control+Z can be used to exit that command.
+
+The command 'exit' exits the program and closes the console (if a new one was
+opened) or returns to the console where the program was started.
+
+
+Character encodings
+-------------------
+
+All HFST tools work in a UTF-8 mode. When hfst-xfst starts, it sets the console
+code point to 65001, i.e. UTF-8. Scriptfiles must be encoded as UTF-8 with no
+BOM (byte order mark) included.
+
+
+Options
+-------
+
+When run from the command line, a number of options are available. For more
+info, see the page <https://kitwiki.csc.fi/twiki/bin/view/KitWiki/HfstXfst>.
+Some important options related to input and output processing are:
+
+ -F, --scriptfile=FILE Read commands from FILE, and quit
+ -l, --startupfile=FILE Read commands from FILE on startup
+ -p, --pipe-mode[=STREAM] Control input and output streams
+
+In interactive mode, all commands are read from console window, i.e. line by
+line from the user and all output is printed to the console. If input comes
+from a file or output is redirected to a file, the option --pipe-mode (or the
+short form -p) must be used:
+
+ type input.xfst | hfst-xfst.exe --pipe-mode=input
+ hfst-xfst.exe --pipe-mode=output > output.txt
+ type input.xfst | hfst-xfst.exe --pipe-mode[=both] > output.txt
+
+(--pipe-mode with no argument is the same as --pipe-mode=both)
+
+
diff --git a/scripts/copy-for-windows.sh b/scripts/copy-for-windows.sh
new file mode 100755
index 0000000..47fc45a
--- /dev/null
+++ b/scripts/copy-for-windows.sh
@@ -0,0 +1,341 @@
+#!/bin/sh
+
+# A script for copying winopenfst (top directory given as the second argument)
+# and foma backends and libhfst/src in a directory given as first argument
+# for native windows compilation.
+
+if [ -d "$1" ]; then
+ echo "Directory $1 exists"
+ exit 1
+else
+ mkdir $1
+fi
+
+#openfstdir=
+#if [ "$2" = "" ]; then
+ openfstdir="back-ends/openfstwin"
+# echo "No second argument given, defaulting to openfst directory '$openfstdir'"
+#else
+# openfstdir=$2
+#fi
+
+mkdir $1/back-ends
+
+# foma back-end
+mkdir $1/back-ends/foma/
+
+for file in \
+apply.c coaccessible.c constructions.c \
+define.c determinize.c dynarray.c \
+extract.c flags.c int_stack.c io.c \
+lex.cmatrix.c lex.yy.c mem.c minimize.c \
+regex.c reverse.c rewrite.c sigma.c \
+spelling.c stringhash.c structures.c \
+topsort.c trie.c utf8.c foma.h fomalib.h \
+fomalibconf.h regex.h;
+do
+ cp back-ends/foma/$file $1/back-ends/foma/;
+done
+
+# openfstwin back-end
+mkdir $1/back-ends/openfstwin
+mkdir $1/back-ends/openfstwin/src
+mkdir $1/back-ends/openfstwin/src/include
+mkdir $1/back-ends/openfstwin/src/include/fst
+mkdir $1/back-ends/openfstwin/src/lib
+
+for file in \
+compat flags fst properties symbol-table symbol-table-ops util;
+do
+ cp $openfstdir/src/lib/$file.cc $1/back-ends/openfstwin/src/lib/$file.cpp
+done
+
+# file symbol-table-ops.cc not in version 1.2.6
+#if [ -f "$openfstdir/src/lib/symbol-table-ops.cc" ]; then
+# cp $openfstdir/src/lib/symbol-table-ops.cc $1/back-ends/openfstwin/src/lib/symbol-table-ops.cpp
+#fi
+
+cp $openfstdir/src/include/fst/*.h $1/back-ends/openfstwin/src/include/fst/
+
+# libhfst/src and subdirectories
+mkdir $1/libhfst
+mkdir $1/libhfst/src
+mkdir $1/libhfst/src/implementations
+mkdir $1/libhfst/src/implementations/compose_intersect
+mkdir $1/libhfst/src/implementations/optimized-lookup
+mkdir $1/libhfst/src/parsers
+
+# libhfst/src without subdirectories
+for file in \
+FormatSpecifiers.h HarmonizeUnknownAndIdentitySymbols.h \
+HfstDataTypes.h HfstEpsilonHandler.h HfstExceptionDefs.h \
+HfstExceptions.h HfstExtractStrings.h HfstFlagDiacritics.h \
+HfstInputStream.h HfstLookupFlagDiacritics.h HfstOutputStream.h \
+HfstSymbolDefs.h HfstTokenizer.h HfstTransducer.h HfstXeroxRules.h \
+hfst.h hfst.hpp.in hfst_apply_schemas.h hfstdll.h;
+do
+ cp libhfst/src/$file $1/libhfst/src/
+done
+
+for file in \
+HarmonizeUnknownAndIdentitySymbols HfstApply HfstDataTypes \
+HfstEpsilonHandler HfstExceptionDefs HfstExceptions HfstFlagDiacritics \
+HfstInputStream HfstLookupFlagDiacritics HfstOutputStream HfstRules \
+HfstSymbolDefs HfstTokenizer HfstTransducer HfstXeroxRules \
+HfstXeroxRulesTest;
+do
+ cp libhfst/src/$file.cc $1/libhfst/src/$file.cpp
+done
+
+# libhfst/src/implementations without subdirectories
+for file in \
+ConvertTransducerFormat.h FomaTransducer.h HfstFastTransitionData.h \
+HfstOlTransducer.h HfstTransition.h HfstTransitionGraph.h \
+HfstTropicalTransducerTransitionData.h LogWeightTransducer.h \
+TropicalWeightTransducer.h;
+do
+ cp libhfst/src/implementations/$file $1/libhfst/src/implementations/
+done
+
+for file in \
+ConvertFomaTransducer ConvertLogWeightTransducer ConvertOlTransducer \
+ConvertTransducerFormat ConvertTropicalWeightTransducer FomaTransducer \
+HfstOlTransducer HfstTransitionGraph HfstTropicalTransducerTransitionData \
+LogWeightTransducer TropicalWeightTransducer;
+do
+ cp libhfst/src/implementations/$file.cc $1/libhfst/src/implementations/$file.cpp
+done
+
+# implementations/compose_intersect
+for file in \
+ComposeIntersectFst ComposeIntersectLexicon ComposeIntersectRule \
+ComposeIntersectRulePair ComposeIntersectUtilities;
+do
+ cp libhfst/src/implementations/compose_intersect/$file.cc \
+ $1/libhfst/src/implementations/compose_intersect/$file.cpp
+ cp libhfst/src/implementations/compose_intersect/$file.h \
+ $1/libhfst/src/implementations/compose_intersect/$file.h
+done
+
+# implementations/optimized-lookup
+for file in \
+convert pmatch transducer;
+do
+ cp libhfst/src/implementations/optimized-lookup/$file.cc \
+ $1/libhfst/src/implementations/optimized-lookup/$file.cpp
+ cp libhfst/src/implementations/optimized-lookup/$file.h \
+ $1/libhfst/src/implementations/optimized-lookup/$file.h
+done
+for file in \
+find_epsilon_loops ospell;
+do
+ cp libhfst/src/implementations/optimized-lookup/$file.cc \
+ $1/libhfst/src/implementations/optimized-lookup/$file.cpp
+done
+
+# parsers
+for file in \
+LexcCompiler PmatchCompiler XreCompiler \
+lexc-utils pmatch_utils xre_utils;
+do
+ cp libhfst/src/parsers/$file.cc \
+ $1/libhfst/src/parsers/$file.cpp
+ cp libhfst/src/parsers/$file.h \
+ $1/libhfst/src/parsers/$file.h
+done
+for file in \
+lexc-lexer pmatch_lex xre_lex;
+do
+ cp libhfst/src/parsers/$file.cc \
+ $1/libhfst/src/parsers/$file.cpp
+done
+for file in \
+lexc-parser pmatch_parse xre_parse;
+do
+ cp libhfst/src/parsers/$file.cc \
+ $1/libhfst/src/parsers/$file.cpp
+ cp libhfst/src/parsers/$file.hh \
+ $1/libhfst/src/parsers/$file.hh
+done
+
+# make scripts and headers
+# cp scripts/make-foma.bat $1/back-ends/foma/
+# cp scripts/make-openfstwin.bat $1/back-ends/openfstwin/src/lib/
+# cp scripts/test-openfstwin.bat $1/back-ends/openfstwin/src/lib/
+# cp scripts/make-parsers.bat $1/libhfst/src/parsers/
+# cp scripts/make-implementations.bat $1/libhfst/src/implementations/
+# cp scripts/make-libhfst.bat $1/libhfst/src/
+# cp scripts/test-libhfst.bat $1/libhfst/src/
+# cp scripts/generate-python-bindings.bat $1/libhfst/src/
+cp scripts/make-python-bindings.bat $1/libhfst/src/
+cp scripts/test_libhfst_win.py $1/libhfst/src/
+cp scripts/libhfst_win.i $1/libhfst/src/
+cp scripts/make-hfst-xfst.bat $1/libhfst/src/
+cp scripts/make-hfst-proc.bat $1/libhfst/src/
+cp scripts/make-hfst-lexc.bat $1/libhfst/src/
+cp scripts/make-hfst-tool.bat $1/libhfst/src/
+cp scripts/make-hfst-tools.bat $1/libhfst/src/
+cp scripts/make-htwolcpre1.bat $1/libhfst/src/
+cp scripts/make-htwolcpre2.bat $1/libhfst/src/
+cp scripts/make-htwolcpre3.bat $1/libhfst/src/
+
+for file in test.lexc test.pmatch test.twolc test.xfst \
+test_xfst_result.txt test_twolc_result.txt test_pmatch_result.txt test_lexc_result.txt;
+do
+ cp scripts/windows_tests/$file $1/libhfst/src/
+done
+
+cp scripts/test-hfst-tools.bat $1/libhfst/src/
+cp scripts/README_xfst.txt $1/libhfst/src/
+cp scripts/README_eight_tools.txt $1/libhfst/src/
+
+# copy missing headers and change some headers included
+cp scripts/stdint.h $1/back-ends/foma/
+cp scripts/inttypes.h $1/back-ends/foma/
+# unistd.h is included in some generated files
+for file in \
+$1/libhfst/src/parsers/pmatch_lex.cpp \
+$1/libhfst/src/parsers/xre_lex.cpp \
+$1/libhfst/src/parsers/lexc-lexer.cpp \
+$1/back-ends/foma/lex.cmatrix.c;
+do
+ sed -i 's/#include <unistd.h>/#include <io.h>/' $file
+done
+sed -i 's/pmatchwrap( )/pmatchwrap(void)/' $1/libhfst/src/parsers/pmatch_lex.cpp
+sed -i 's/hlexcwrap( )/hlexcwrap(void)/' $1/libhfst/src/parsers/lexc-lexer.cpp
+
+# copy files for tools
+
+# create subdirectories
+mkdir $1/tools
+mkdir $1/tools/src
+mkdir $1/tools/src/parsers
+mkdir $1/tools/src/hfst-proc
+mkdir $1/tools/src/inc
+mkdir $1/tools/src/hfst-twolc
+mkdir $1/tools/src/hfst-twolc/src
+mkdir $1/tools/src/hfst-twolc/src/alphabet_src
+mkdir $1/tools/src/hfst-twolc/src/commandline_src
+mkdir $1/tools/src/hfst-twolc/src/io_src
+mkdir $1/tools/src/hfst-twolc/src/rule_src
+mkdir $1/tools/src/hfst-twolc/src/string_src
+mkdir $1/tools/src/hfst-twolc/src/variable_src
+
+
+for file in \
+xfst-utils XfstCompiler hfst-xfst xfst-parser xfst-lexer \
+init_help xfst_help_message;
+do
+ cp tools/src/parsers/$file.cc $1/tools/src/parsers/$file.cpp
+done
+
+cp tools/src/parsers/xfst_help_message.h $1/tools/src/parsers/
+
+sed -i 's/#include <unistd.h>/#include <io.h>/' $1/tools/src/parsers/xfst-lexer.cpp
+sed -i 's/hxfstwrap( )/hxfstwrap(void)/' $1/tools/src/parsers/xfst-lexer.cpp
+#sed -i 's/#include "help_message.cc"/#include "help_message.cpp"/' $1/tools/src/parsers/XfstCompiler.cpp
+
+
+# compare, strings2fst and txt2fst are needed for testing hfst-xfst
+for file in \
+hfst-program-options hfst-commandline hfst-tool-metadata HfstStrings2FstTokenizer \
+hfst-file-to-mem hfst-string-conversions hfst-getopt \
+hfst-lexc-compiler hfst-compare hfst-strings2fst hfst-txt2fst hfst-pmatch hfst-pmatch2fst \
+hfst-lookup hfst-optimized-lookup;
+do
+ cp tools/src/$file.cc $1/tools/src/$file.cpp
+done
+
+# hfst-proc
+for file in \
+hfst-proc formatter lookup-path lookup-state tokenizer transducer applicators alphabet;
+do
+ cp tools/src/hfst-proc/$file.cc $1/tools/src/hfst-proc/$file.cpp
+done
+
+for file in \
+hfst-proc.h formatter.h lookup-path.h lookup-state.h tokenizer.h \
+transducer.h buffer.h applicators.h alphabet.h;
+do
+ cp tools/src/hfst-proc/$file $1/tools/src/hfst-proc/
+done
+
+for file in \
+hfst-commandline.h hfst-program-options.h hfst-tool-metadata.h \
+HfstStrings2FstTokenizer.h hfst-string-conversions.h \
+hfst-file-to-mem.h hfst-getopt.h hfst-optimized-lookup.h;
+do
+ cp tools/src/$file $1/tools/src/
+done
+
+for file in \
+XfstCompiler.h xfst-utils.h xfst-parser.hh cmd.h abbrcmd.h;
+do
+ cp tools/src/parsers/$file $1/tools/src/parsers/
+done
+
+for file in \
+check-params-binary.h check-params-common.h check-params-unary.h getopt-cases-binary.h \
+getopt-cases-common.h getopt-cases-error.h getopt-cases-unary.h globals-binary.h \
+globals-common.h globals-unary.h;
+do
+ cp tools/src/inc/$file $1/tools/src/inc/
+done
+
+
+# Copy twolc
+
+TWOLC_DIR=tools/src/hfst-twolc/src
+
+for file in \
+HfstTwolcDefs.h common_globals.h grammar_defs.h hfst-twolc.bat \
+htwolcpre1.hh htwolcpre2.hh htwolcpre3.hh;
+do
+ cp $TWOLC_DIR/$file $1/$TWOLC_DIR/
+done
+
+for file in \
+hfst-twolc-system htwolcpre1 htwolcpre2 htwolcpre3 scanner1 scanner2 scanner3;
+do
+ cp $TWOLC_DIR/$file.cc $1/$TWOLC_DIR/$file.cpp
+done
+
+for file in scanner1.cpp scanner2.cpp scanner3.cpp;
+do
+ sed -i 's/#include <unistd.h>/#include <io.h>/' $1/$TWOLC_DIR/$file
+ sed -i 's/yywrap( )/yywrap(void)/' $1/$TWOLC_DIR/$file
+done
+
+cp $TWOLC_DIR/alphabet_src/Alphabet.cc $1/$TWOLC_DIR/alphabet_src/Alphabet.cpp
+cp $TWOLC_DIR/alphabet_src/Alphabet.h $1/$TWOLC_DIR/alphabet_src/Alphabet.h
+cp $TWOLC_DIR/commandline_src/CommandLine.cc $1/$TWOLC_DIR/commandline_src/CommandLine.cpp
+cp $TWOLC_DIR/commandline_src/CommandLine.h $1/$TWOLC_DIR/commandline_src/CommandLine.h
+cp $TWOLC_DIR/io_src/InputReader.cc $1/$TWOLC_DIR/io_src/InputReader.cpp
+cp $TWOLC_DIR/io_src/InputReader.h $1/$TWOLC_DIR/io_src/InputReader.h
+cp $TWOLC_DIR/io_src/input_defs.h $1/$TWOLC_DIR/io_src/input_defs.h
+
+for file in ConflictResolvingLeftArrowRule ConflictResolvingRightArrowRule \
+LeftArrowRule LeftArrowRuleContainer LeftRestrictionArrowRule OtherSymbolTransducer \
+RightArrowRule RightArrowRuleContainer Rule RuleContainer TwolCGrammar;
+do
+ cp $TWOLC_DIR/rule_src/$file.h $1/$TWOLC_DIR/rule_src/$file.h
+ cp $TWOLC_DIR/rule_src/$file.cc $1/$TWOLC_DIR/rule_src/$file.cpp
+done
+
+cp $TWOLC_DIR/string_src/string_manipulation.cc $1/$TWOLC_DIR/string_src/string_manipulation.cpp
+cp $TWOLC_DIR/string_src/string_manipulation.h $1/$TWOLC_DIR/string_src/string_manipulation.h
+
+for file in ConstContainerIterator.h MatchedConstContainerIterator.h MixedConstContainerIterator.h \
+RuleSymbolVector.h RuleVariables.h RuleVariablesConstIterator.h VariableBlock.h VariableBlockContainer.h \
+VariableContainer.h VariableContainerBase.h VariableDefs.h VariableValueIterator.h VariableValues.h;
+do
+ cp $TWOLC_DIR/variable_src/$file $1/$TWOLC_DIR/variable_src/
+done
+
+for file in RuleSymbolVector RuleVariables RuleVariablesConstIterator VariableValues;
+do
+ cp $TWOLC_DIR/variable_src/$file.cc $1/$TWOLC_DIR/variable_src/$file.cpp
+done
+
+# For convenience
+cp $TWOLC_DIR/hfst-twolc.bat $1/libhfst/src/
diff --git a/scripts/generate-python-bindings.bat b/scripts/generate-python-bindings.bat
new file mode 100644
index 0000000..f17e055
--- /dev/null
+++ b/scripts/generate-python-bindings.bat
@@ -0,0 +1,4 @@
+C:\swigwin-3.0.5\swig.exe -python -c++ -I"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" -Wall -o _libhfst.cpp libhfst_win.i
+cl.exe /EHsc /LD /IC:\Python33\include\ _libhfst.cpp /link libhfst.lib
+move _libhfst.dll _libhfst.pyd
+copy C:\python33\libs\python33.lib .
diff --git a/scripts/generate-static-binaries.sh b/scripts/generate-static-binaries.sh
new file mode 100755
index 0000000..7a1d97d
--- /dev/null
+++ b/scripts/generate-static-binaries.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+#
+# A script for generating static binaries for eight tools:
+# hfst-xfst, hfst-lexc, hfst-lookup, hfst-optimized-lookup,
+# hfst-pmatch, hfst-pmatch2fst, hfst-twolc and hfst-proc.
+#
+
+# recognize whther we are on Linux or Mac
+ENVIRONMENT= # linux/mac
+SHOW_LINKS= # ldd/otool -L
+CONFIGURE_ARGS='--enable-xfst --with-readline --enable-foma-wrapper=no'
+DYLIB_EXTENSION= # so/dylib
+if (uname -a | grep 'Linux' > /dev/null); then
+ ENVIRONMENT='Linux';
+ SHOW_LINKS='ldd';
+# CONFIGURE_ARGS are ok as such
+ DYLIB_EXTENSION='so';
+ echo "generating for Linux environment"
+elif (uname -a | grep 'Darwin' > /dev/null); then
+ ENVIRONMENT='Mac';
+ SHOW_LINKS='otool -L'
+ CONFIGURE_ARGS=$CONFIGURE_ARGS' --disable-dependency-tracking CFLAGS="-arch i386 -arch x86_64" CXXFLAGS="-arch i386 -arch x86_64" LDFLAGS="-arch i386 -arch x86_64"'
+ DYLIB_EXTENSION='dylib'
+ echo "generating for Mac environment"
+else
+ echo "Unknown environment, exiting program"
+ exit 1
+fi
+
+# compile tool given as first argument and link it statically to libhfst
+compile_statically ()
+{
+ echo "compiling "$1"..."
+ # compile tool normally but save the commands in log file
+ make LDFLAGS="-static-libstdc++" $1 > LOG
+ echo "linking "$1" statically..."
+ # tweak the link command to make libhfst linking static
+ grep 'libtool: link:' LOG | sed 's/libhfst.'$DYLIB_EXTENSION'/libhfst.a/; s/libtool: link: //' | sh -v
+ if ($SHOW_LINKS .libs/$1 | grep 'libhfst'); then
+ echo "static linking of libhfst failed foor tool "$1
+ exit 1
+ fi
+}
+
+
+autoreconf -i &&
+./configure $CONFIGURE_ARGS && scripts/generate-cc-files.sh
+
+# these are needed for compiling individual tools
+cd back-ends && make LDFLAGS="-static-libstdc++" && cd ../libhfst/src && make LDFLAGS="-static-libstdc++"
+
+# compile all eight tools
+cd ../../tools/src/parsers &&
+compile_statically hfst-xfst &&
+cd ../../../tools/src &&
+compile_statically hfst-lexc &&
+compile_statically hfst-lookup &&
+compile_statically hfst-optimized-lookup &&
+compile_statically hfst-pmatch &&
+compile_statically hfst-pmatch2fst &&
+cd ../../tools/src/hfst-proc &&
+compile_statically hfst-apertium-proc &&
+cd ../../../tools/src/hfst-twolc/src &&
+compile_statically htwolcpre1 &&
+compile_statically htwolcpre2 &&
+compile_statically htwolcpre3
+cd ../../../..
diff --git a/scripts/hfst-twolc-bin b/scripts/hfst-twolc-bin
new file mode 100755
index 0000000..686cb5f
--- /dev/null
+++ b/scripts/hfst-twolc-bin
@@ -0,0 +1,5 @@
+#!/bin/sh
+BINDIR=`echo $0 | sed 's/hfst-twolc//'`
+$BINDIR/htwolcpre1 $@ |
+$BINDIR/htwolcpre2 $@ |
+$BINDIR/htwolcpre3 $@
diff --git a/scripts/inttypes.h b/scripts/inttypes.h
new file mode 100644
index 0000000..2554277
--- /dev/null
+++ b/scripts/inttypes.h
@@ -0,0 +1,305 @@
+// ISO C9x compliant inttypes.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
+//
+// Copyright (c) 2006 Alexander Chemeris
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. The name of the author may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_INTTYPES_H_ // [
+#define _MSC_INTTYPES_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include "stdint.h"
+
+// 7.8 Format conversion of integer types
+
+typedef struct {
+ intmax_t quot;
+ intmax_t rem;
+} imaxdiv_t;
+
+// 7.8.1 Macros for format specifiers
+
+#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198
+
+// The fprintf macros for signed integers are:
+#define PRId8 "d"
+#define PRIi8 "i"
+#define PRIdLEAST8 "d"
+#define PRIiLEAST8 "i"
+#define PRIdFAST8 "d"
+#define PRIiFAST8 "i"
+
+#define PRId16 "hd"
+#define PRIi16 "hi"
+#define PRIdLEAST16 "hd"
+#define PRIiLEAST16 "hi"
+#define PRIdFAST16 "hd"
+#define PRIiFAST16 "hi"
+
+#define PRId32 "I32d"
+#define PRIi32 "I32i"
+#define PRIdLEAST32 "I32d"
+#define PRIiLEAST32 "I32i"
+#define PRIdFAST32 "I32d"
+#define PRIiFAST32 "I32i"
+
+#define PRId64 "I64d"
+#define PRIi64 "I64i"
+#define PRIdLEAST64 "I64d"
+#define PRIiLEAST64 "I64i"
+#define PRIdFAST64 "I64d"
+#define PRIiFAST64 "I64i"
+
+#define PRIdMAX "I64d"
+#define PRIiMAX "I64i"
+
+#define PRIdPTR "Id"
+#define PRIiPTR "Ii"
+
+// The fprintf macros for unsigned integers are:
+#define PRIo8 "o"
+#define PRIu8 "u"
+#define PRIx8 "x"
+#define PRIX8 "X"
+#define PRIoLEAST8 "o"
+#define PRIuLEAST8 "u"
+#define PRIxLEAST8 "x"
+#define PRIXLEAST8 "X"
+#define PRIoFAST8 "o"
+#define PRIuFAST8 "u"
+#define PRIxFAST8 "x"
+#define PRIXFAST8 "X"
+
+#define PRIo16 "ho"
+#define PRIu16 "hu"
+#define PRIx16 "hx"
+#define PRIX16 "hX"
+#define PRIoLEAST16 "ho"
+#define PRIuLEAST16 "hu"
+#define PRIxLEAST16 "hx"
+#define PRIXLEAST16 "hX"
+#define PRIoFAST16 "ho"
+#define PRIuFAST16 "hu"
+#define PRIxFAST16 "hx"
+#define PRIXFAST16 "hX"
+
+#define PRIo32 "I32o"
+#define PRIu32 "I32u"
+#define PRIx32 "I32x"
+#define PRIX32 "I32X"
+#define PRIoLEAST32 "I32o"
+#define PRIuLEAST32 "I32u"
+#define PRIxLEAST32 "I32x"
+#define PRIXLEAST32 "I32X"
+#define PRIoFAST32 "I32o"
+#define PRIuFAST32 "I32u"
+#define PRIxFAST32 "I32x"
+#define PRIXFAST32 "I32X"
+
+#define PRIo64 "I64o"
+#define PRIu64 "I64u"
+#define PRIx64 "I64x"
+#define PRIX64 "I64X"
+#define PRIoLEAST64 "I64o"
+#define PRIuLEAST64 "I64u"
+#define PRIxLEAST64 "I64x"
+#define PRIXLEAST64 "I64X"
+#define PRIoFAST64 "I64o"
+#define PRIuFAST64 "I64u"
+#define PRIxFAST64 "I64x"
+#define PRIXFAST64 "I64X"
+
+#define PRIoMAX "I64o"
+#define PRIuMAX "I64u"
+#define PRIxMAX "I64x"
+#define PRIXMAX "I64X"
+
+#define PRIoPTR "Io"
+#define PRIuPTR "Iu"
+#define PRIxPTR "Ix"
+#define PRIXPTR "IX"
+
+// The fscanf macros for signed integers are:
+#define SCNd8 "d"
+#define SCNi8 "i"
+#define SCNdLEAST8 "d"
+#define SCNiLEAST8 "i"
+#define SCNdFAST8 "d"
+#define SCNiFAST8 "i"
+
+#define SCNd16 "hd"
+#define SCNi16 "hi"
+#define SCNdLEAST16 "hd"
+#define SCNiLEAST16 "hi"
+#define SCNdFAST16 "hd"
+#define SCNiFAST16 "hi"
+
+#define SCNd32 "ld"
+#define SCNi32 "li"
+#define SCNdLEAST32 "ld"
+#define SCNiLEAST32 "li"
+#define SCNdFAST32 "ld"
+#define SCNiFAST32 "li"
+
+#define SCNd64 "I64d"
+#define SCNi64 "I64i"
+#define SCNdLEAST64 "I64d"
+#define SCNiLEAST64 "I64i"
+#define SCNdFAST64 "I64d"
+#define SCNiFAST64 "I64i"
+
+#define SCNdMAX "I64d"
+#define SCNiMAX "I64i"
+
+#ifdef _WIN64 // [
+# define SCNdPTR "I64d"
+# define SCNiPTR "I64i"
+#else // _WIN64 ][
+# define SCNdPTR "ld"
+# define SCNiPTR "li"
+#endif // _WIN64 ]
+
+// The fscanf macros for unsigned integers are:
+#define SCNo8 "o"
+#define SCNu8 "u"
+#define SCNx8 "x"
+#define SCNX8 "X"
+#define SCNoLEAST8 "o"
+#define SCNuLEAST8 "u"
+#define SCNxLEAST8 "x"
+#define SCNXLEAST8 "X"
+#define SCNoFAST8 "o"
+#define SCNuFAST8 "u"
+#define SCNxFAST8 "x"
+#define SCNXFAST8 "X"
+
+#define SCNo16 "ho"
+#define SCNu16 "hu"
+#define SCNx16 "hx"
+#define SCNX16 "hX"
+#define SCNoLEAST16 "ho"
+#define SCNuLEAST16 "hu"
+#define SCNxLEAST16 "hx"
+#define SCNXLEAST16 "hX"
+#define SCNoFAST16 "ho"
+#define SCNuFAST16 "hu"
+#define SCNxFAST16 "hx"
+#define SCNXFAST16 "hX"
+
+#define SCNo32 "lo"
+#define SCNu32 "lu"
+#define SCNx32 "lx"
+#define SCNX32 "lX"
+#define SCNoLEAST32 "lo"
+#define SCNuLEAST32 "lu"
+#define SCNxLEAST32 "lx"
+#define SCNXLEAST32 "lX"
+#define SCNoFAST32 "lo"
+#define SCNuFAST32 "lu"
+#define SCNxFAST32 "lx"
+#define SCNXFAST32 "lX"
+
+#define SCNo64 "I64o"
+#define SCNu64 "I64u"
+#define SCNx64 "I64x"
+#define SCNX64 "I64X"
+#define SCNoLEAST64 "I64o"
+#define SCNuLEAST64 "I64u"
+#define SCNxLEAST64 "I64x"
+#define SCNXLEAST64 "I64X"
+#define SCNoFAST64 "I64o"
+#define SCNuFAST64 "I64u"
+#define SCNxFAST64 "I64x"
+#define SCNXFAST64 "I64X"
+
+#define SCNoMAX "I64o"
+#define SCNuMAX "I64u"
+#define SCNxMAX "I64x"
+#define SCNXMAX "I64X"
+
+#ifdef _WIN64 // [
+# define SCNoPTR "I64o"
+# define SCNuPTR "I64u"
+# define SCNxPTR "I64x"
+# define SCNXPTR "I64X"
+#else // _WIN64 ][
+# define SCNoPTR "lo"
+# define SCNuPTR "lu"
+# define SCNxPTR "lx"
+# define SCNXPTR "lX"
+#endif // _WIN64 ]
+
+#endif // __STDC_FORMAT_MACROS ]
+
+// 7.8.2 Functions for greatest-width integer types
+
+// 7.8.2.1 The imaxabs function
+#define imaxabs _abs64
+
+// 7.8.2.2 The imaxdiv function
+
+// This is modified version of div() function from Microsoft's div.c found
+// in %MSVC.NET%\crt\src\div.c
+#ifdef STATIC_IMAXDIV // [
+static
+#else // STATIC_IMAXDIV ][
+_inline
+#endif // STATIC_IMAXDIV ]
+imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom)
+{
+ imaxdiv_t result;
+
+ result.quot = numer / denom;
+ result.rem = numer % denom;
+
+ if (numer < 0 && result.rem > 0) {
+ // did division wrong; must fix up
+ ++result.quot;
+ result.rem -= denom;
+ }
+
+ return result;
+}
+
+// 7.8.2.3 The strtoimax and strtoumax functions
+#define strtoimax _strtoi64
+#define strtoumax _strtoui64
+
+// 7.8.2.4 The wcstoimax and wcstoumax functions
+#define wcstoimax _wcstoi64
+#define wcstoumax _wcstoui64
+
+
+#endif // _MSC_INTTYPES_H_ ]
diff --git a/scripts/libhfst_win.i b/scripts/libhfst_win.i
new file mode 100644
index 0000000..ab3d6fb
--- /dev/null
+++ b/scripts/libhfst_win.i
@@ -0,0 +1,222 @@
+%module libhfst
+%include "std_string.i"
+%include "std_vector.i"
+%include "std_pair.i"
+%include "std_set.i"
+
+%{
+#define HFSTIMPORT
+#include "HfstDataTypes.h"
+#include "HfstTransducer.h"
+#include "parsers/XreCompiler.h"
+#include "implementations/HfstTransitionGraph.h"
+
+// todo instead: #include "hfst_extensions.h"
+
+namespace hfst {
+
+typedef std::vector<float> FloatVector;
+
+hfst::ImplementationType type = hfst::TROPICAL_OPENFST_TYPE;
+
+hfst::HfstTokenizer deftok; // default tokenizer
+
+void set_default_fst_type(hfst::ImplementationType t)
+{
+ type = t;
+}
+
+hfst::ImplementationType get_default_fst_type()
+{
+ return type;
+}
+
+std::string fst_type_to_string(hfst::ImplementationType t)
+{
+ std::string retval = hfst::implementation_type_to_string(t);
+ return retval;
+}
+
+hfst::HfstTransducer fst(const std::string & symbol)
+{
+ return hfst::HfstTransducer(symbol, type);
+}
+
+hfst::HfstTransducer fst(const std::string & isymbol, const std::string & osymbol)
+{
+ return hfst::HfstTransducer(isymbol, osymbol, type);
+}
+
+hfst::HfstTransducer * regex(const std::string & regex_string)
+{
+ hfst::xre::XreCompiler comp(type);
+ return comp.compile(regex_string);
+}
+
+hfst::HfstTransducer word(const std::string & w, float weight=0)
+{
+ return hfst::HfstTransducer(w, deftok, type);
+}
+
+hfst::HfstTransducer word_pair(const std::string & wi, const std::string & wo, float weight=0)
+{
+ return hfst::HfstTransducer(wi, wo, deftok, type);
+}
+
+hfst::HfstTransducer word_list_hfst(const StringVector & wl, const FloatVector & weights)
+{
+ hfst::implementations::HfstBasicTransducer retval;
+ unsigned int i=0;
+ for (std::vector<std::string>::const_iterator it = wl.begin();
+ it != wl.end(); it++)
+ {
+ retval.disjunct(deftok.tokenize(*it), weights[i]);
+ i++;
+ }
+ return hfst::HfstTransducer(retval, type);
+}
+
+hfst::HfstTransducer word_pair_list_hfst(const StringPairVector & wpl, const FloatVector & weights)
+{
+ hfst::implementations::HfstBasicTransducer retval;
+ unsigned int i=0;
+ for (std::vector<std::pair<std::string, std::string> >::const_iterator it = wpl.begin();
+ it != wpl.end(); it++)
+ {
+ retval.disjunct(deftok.tokenize(it->first, it->second), weights[i]);
+ i++;
+ }
+ return hfst::HfstTransducer(retval, type);
+}
+
+}
+
+%}
+
+%include <windows.h>
+
+%include "typemaps.i"
+
+namespace std {
+%template(StringVector) vector<string>;
+%template(StringPair) pair<string, string>;
+%template(StringPairVector) vector<pair<string, string > >;
+%template(FloatVector) vector<float>;
+%template(StringSet) set<string>;
+//%template(HfstOneLevelPath) pair<float, vector<string> >;
+//%template(HfstOneLevelPaths) set<pair<float, vector<string> > >;
+}
+
+%ignore hfst::HfstTransducer::lookup_fd(const std::string & s) const;
+
+namespace hfst
+{
+
+typedef std::vector<std::string> StringVector;
+typedef std::pair<std::string, std::string> StringPair;
+typedef std::vector<std::pair<std::string, std::string> > StringPairVector;
+typedef std::vector<float> FloatVector;
+typedef std::set<std::string> StringSet;
+typedef std::pair<float, std::vector<std::string> > HfstOneLevelPath;
+typedef std::set<std::pair<float, std::vector<std::string> > > HfstOneLevelPaths;
+
+enum ImplementationType
+{
+ SFST_TYPE,
+ TROPICAL_OPENFST_TYPE,
+ LOG_OPENFST_TYPE,
+ FOMA_TYPE,
+ XFSM_TYPE,
+ HFST_OL_TYPE,
+ HFST_OLW_TYPE,
+ HFST2_TYPE,
+ UNSPECIFIED_TYPE,
+ ERROR_TYPE
+};
+
+%typemap(out) HfstOneLevelPaths* {
+ $result = PyList_New((*$1).size());
+ unsigned int i = 0;
+ for (hfst::HfstOneLevelPaths::const_iterator it = (*$1).begin(); it != (*$1).end(); it++)
+ {
+ std::string result_string("");
+ for (hfst::StringVector::const_iterator svit = it->second.begin(); svit != it->second.end(); svit++)
+ {
+ result_string += *svit;
+ }
+ PyObject * res = PyTuple_New(2);
+ PyTuple_SetItem(res, 0, PyString_FromString(result_string.c_str()));
+ PyTuple_SetItem(res, 1, PyFloat_FromDouble(it->first));
+ PyList_SetItem($result, i, res);
+ i++;
+ }
+}
+
+class HfstTransducer
+{
+public:
+HfstTransducer(ImplementationType);
+HfstTransducer(const std::string &, const std::string &, ImplementationType);
+~HfstTransducer();
+HfstTransducer & concatenate(const HfstTransducer&, bool harmonize=true);
+void write_in_att_format(const std::string &, bool write_weights=true) const;
+
+HfstOneLevelPaths * lookup_fd(const std::string & s, int limit=-1) const;
+
+ %extend {
+ char *__str__() {
+ static char tmp[1024];
+ $self->write_in_att_format(tmp);
+ return tmp;
+ }
+ };
+
+
+};
+
+hfst::HfstTransducer fst(const std::string & symbol);
+hfst::HfstTransducer fst(const std::string & isymbol, const std::string & osymbol);
+hfst::HfstTransducer * regex(const std::string & regex_string);
+hfst::HfstTransducer word(const std::string & w, float weight=0);
+//hfst::HfstTransducer word(const std::string & w, const hfst::HfstTokenizer & tok);
+hfst::HfstTransducer word_pair(const std::string & wi, const std::string & wo, float weight=0);
+//hfst::HfstTransducer word_pair(const std::string & wi, const std::string & wo, const hfst::HfstTokenizer & tok);
+hfst::HfstTransducer word_list_hfst(const StringVector & wl, const FloatVector & weights);
+hfst::HfstTransducer word_pair_list_hfst(const StringPairVector & wpl, const FloatVector & weights);
+
+
+%pythoncode %{
+
+EPSILON='@_EPSILON_SYMBOL_@'
+UNKNOWN='@_UNKNOWN_SYMBOL_@'
+IDENTITY='@_IDENTITY_SYMBOL_@'
+
+def word_list(l):
+ v = StringVector()
+ f = FloatVector()
+ for word in l:
+ v.push_back(word[0])
+ if (len(word) > 1):
+ f.push_back(word[1])
+ else:
+ f.push_back(0)
+ return _libhfst.word_list_hfst(v,f)
+
+def word_pair_list(l):
+ v = StringPairVector()
+ f = FloatVector()
+ for word in l:
+ v.push_back(StringPair(word[0],word[1]))
+ if (len(word) > 2):
+ f.push_back(word[2])
+ else:
+ f.push_back(0)
+ return _libhfst.word_pair_list_hfst(v,f)
+
+%}
+
+void set_default_fst_type(hfst::ImplementationType t);
+hfst::ImplementationType get_default_fst_type();
+std::string fst_type_to_string(hfst::ImplementationType t);
+
+}
diff --git a/scripts/make-foma.bat b/scripts/make-foma.bat
new file mode 100644
index 0000000..65e2774
--- /dev/null
+++ b/scripts/make-foma.bat
@@ -0,0 +1,5 @@
+SET debug_compile=
+
+if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /MDd
+
+cl /EHsc /LD /D_MSC_VER /I . /Felibfoma.dll *.c
diff --git a/scripts/make-hfst-lexc.bat b/scripts/make-hfst-lexc.bat
new file mode 100644
index 0000000..2c59832
--- /dev/null
+++ b/scripts/make-hfst-lexc.bat
@@ -0,0 +1,87 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fehfst-lexc.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src ^
+..\..\tools\src\hfst-commandline.cpp ^
+..\..\tools\src\hfst-file-to-mem.cpp ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\hfst-lexc-compiler.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-hfst-proc.bat b/scripts/make-hfst-proc.bat
new file mode 100644
index 0000000..b5831ca
--- /dev/null
+++ b/scripts/make-hfst-proc.bat
@@ -0,0 +1,97 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+if exist "..\..\tools\src\hfst-proc\transducer.cpp" echo Renaming hfst-proc's transducer.cpp into _transducer.cpp as there is a file with the same name in optimized-lookup && ^
+move ..\..\tools\src\hfst-proc\transducer.cpp ..\..\tools\src\hfst-proc\_transducer.cpp
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fehfst-proc.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src ^
+..\..\tools\src\hfst-commandline.cpp ^
+..\..\tools\src\hfst-file-to-mem.cpp ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\hfst-proc\hfst-proc.cpp ^
+..\..\tools\src\hfst-proc\formatter.cpp ^
+..\..\tools\src\hfst-proc\lookup-path.cpp ^
+..\..\tools\src\hfst-proc\lookup-state.cpp ^
+..\..\tools\src\hfst-proc\tokenizer.cpp ^
+..\..\tools\src\hfst-proc\_transducer.cpp ^
+..\..\tools\src\hfst-proc\applicators.cpp ^
+..\..\tools\src\hfst-proc\alphabet.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-hfst-tool.bat b/scripts/make-hfst-tool.bat
new file mode 100644
index 0000000..ee2d15c
--- /dev/null
+++ b/scripts/make-hfst-tool.bat
@@ -0,0 +1,87 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fe%1.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src ^
+..\..\tools\src\hfst-commandline.cpp ^
+..\..\tools\src\hfst-file-to-mem.cpp ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\%1.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-hfst-tools.bat b/scripts/make-hfst-tools.bat
new file mode 100644
index 0000000..44d6870
--- /dev/null
+++ b/scripts/make-hfst-tools.bat
@@ -0,0 +1 @@
+make-hfst-xfst.bat && make-hfst-lexc.bat && make-htwolcpre1.bat && make-htwolcpre2.bat && make-htwolcpre3.bat && make-hfst-proc.bat && make-hfst-tool.bat hfst-optimized-lookup && make-hfst-tool.bat hfst-lookup && make-hfst-tool.bat hfst-pmatch && make-hfst-tool.bat hfst-pmatch2fst && make-hfst-tool.bat hfst-compare && make-hfst-tool.bat hfst-strings2fst && make-hfst-tool.bat hfst-txt2fst
diff --git a/scripts/make-hfst-xfst.bat b/scripts/make-hfst-xfst.bat
new file mode 100644
index 0000000..7c15f7e
--- /dev/null
+++ b/scripts/make-hfst-xfst.bat
@@ -0,0 +1,92 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fehfst-xfst.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src /I..\..\tools\src\parsers ^
+..\..\tools\src\hfst-commandline.cpp ^
+..\..\tools\src\hfst-file-to-mem.cpp ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\parsers\hfst-xfst.cpp ^
+..\..\tools\src\parsers\XfstCompiler.cpp ^
+..\..\tools\src\parsers\xfst-lexer.cpp ^
+..\..\tools\src\parsers\xfst-parser.cpp ^
+..\..\tools\src\parsers\xfst-utils.cpp ^
+..\..\tools\src\parsers\xfst_help_message.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-htwolcpre1.bat b/scripts/make-htwolcpre1.bat
new file mode 100644
index 0000000..8e03551
--- /dev/null
+++ b/scripts/make-htwolcpre1.bat
@@ -0,0 +1,95 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fehtwolcpre1.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src ^
+/I..\..\tools\src\hfst-twolc\src /I..\..\tools\src\hfst-twolc\src\commandline_src /I..\..\tools\src\hfst-twolc\src\io_src ^
+/I..\..\tools\src\hfst-twolc\src\variable_src /I..\..\tools\src\hfst-twolc\src\string_src ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\hfst-twolc\src\htwolcpre1.cpp ^
+..\..\tools\src\hfst-twolc\src\scanner1.cpp ^
+..\..\tools\src\hfst-twolc\src\commandline_src\CommandLine.cpp ^
+..\..\tools\src\hfst-twolc\src\io_src\InputReader.cpp ^
+..\..\tools\src\hfst-twolc\src\string_src\string_manipulation.cpp ^
+..\..\tools\src\hfst-twolc\src\variable_src\VariableValues.cpp ^
+..\..\tools\src\hfst-twolc\src\variable_src\RuleVariablesConstIterator.cpp ^
+..\..\tools\src\hfst-twolc\src\variable_src\RuleVariables.cpp ^
+..\..\tools\src\hfst-twolc\src\variable_src\RuleSymbolVector.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-htwolcpre2.bat b/scripts/make-htwolcpre2.bat
new file mode 100644
index 0000000..f7795ba
--- /dev/null
+++ b/scripts/make-htwolcpre2.bat
@@ -0,0 +1,89 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fehtwolcpre2.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src ^
+/I..\..\tools\src\hfst-twolc\src /I..\..\tools\src\hfst-twolc\src\commandline_src /I..\..\tools\src\hfst-twolc\src\io_src ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\hfst-twolc\src\htwolcpre2.cpp ^
+..\..\tools\src\hfst-twolc\src\scanner2.cpp ^
+..\..\tools\src\hfst-twolc\src\commandline_src\CommandLine.cpp ^
+..\..\tools\src\hfst-twolc\src\io_src\InputReader.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-htwolcpre3.bat b/scripts/make-htwolcpre3.bat
new file mode 100644
index 0000000..db95b41
--- /dev/null
+++ b/scripts/make-htwolcpre3.bat
@@ -0,0 +1,103 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst && ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+cl /EHsc /Zc:wchar_t /Fehtwolcpre3.exe ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I..\..\tools\src ^
+/I..\..\tools\src\hfst-twolc\src /I..\..\tools\src\hfst-twolc\src\commandline_src /I..\..\tools\src\hfst-twolc\src\io_src ^
+/I..\..\tools\src\hfst-twolc\src\alphabet_src /I..\..\tools\src\hfst-twolc\src\rule_src /I..\..\tools\src\hfst-twolc\src\string_src ^
+..\..\tools\src\hfst-program-options.cpp ^
+..\..\tools\src\hfst-string-conversions.cpp ^
+..\..\tools\src\HfstStrings2FstTokenizer.cpp ^
+..\..\tools\src\hfst-tool-metadata.cpp ^
+..\..\tools\src\hfst-getopt.cpp ^
+..\..\tools\src\hfst-twolc\src\htwolcpre3.cpp ^
+..\..\tools\src\hfst-twolc\src\scanner3.cpp ^
+..\..\tools\src\hfst-twolc\src\commandline_src\CommandLine.cpp ^
+..\..\tools\src\hfst-twolc\src\io_src\InputReader.cpp ^
+..\..\tools\src\hfst-twolc\src\string_src\string_manipulation.cpp ^
+..\..\tools\src\hfst-twolc\src\alphabet_src\Alphabet.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\ConflictResolvingLeftArrowRule.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\ConflictResolvingRightArrowRule.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\LeftArrowRule.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\LeftArrowRuleContainer.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\LeftRestrictionArrowRule.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\OtherSymbolTransducer.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\RightArrowRule.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\RightArrowRuleContainer.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\Rule.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\RuleContainer.cpp ^
+..\..\tools\src\hfst-twolc\src\rule_src\TwolCGrammar.cpp ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c user32.lib
+
+
diff --git a/scripts/make-implementations.bat b/scripts/make-implementations.bat
new file mode 100644
index 0000000..516053a
--- /dev/null
+++ b/scripts/make-implementations.bat
@@ -0,0 +1,22 @@
+cl /c /EHsc /I..\..\..\libhfst\src /I..\..\..\back-ends\foma /I..\..\..\back-ends /I..\..\..\back-ends\openfstwin\src\include /I ..\..\.. ^
+HfstTransitionGraph.cpp ^
+ConvertTransducerFormat.cpp ^
+HfstTropicalTransducerTransitionData.cpp ^
+ConvertTropicalWeightTransducer.cpp ^
+ConvertLogWeightTransducer.cpp ^
+ConvertFomaTransducer.cpp ^
+ConvertOlTransducer.cpp ^
+TropicalWeightTransducer.cpp ^
+LogWeightTransducer.cpp ^
+FomaTransducer.cpp ^
+HfstOlTransducer.cpp ^
+compose_intersect\ComposeIntersectRulePair.cpp ^
+compose_intersect\ComposeIntersectLexicon.cpp ^
+compose_intersect\ComposeIntersectRule.cpp ^
+compose_intersect\ComposeIntersectFst.cpp ^
+compose_intersect\ComposeIntersectUtilities.cpp ^
+optimized-lookup\transducer.cpp ^
+optimized-lookup\convert.cpp ^
+optimized-lookup\ospell.cpp ^
+optimized-lookup\pmatch.cpp ^
+optimized-lookup\find_epsilon_loops.cpp
diff --git a/scripts/make-libhfst.bat b/scripts/make-libhfst.bat
new file mode 100644
index 0000000..8229a4b
--- /dev/null
+++ b/scripts/make-libhfst.bat
@@ -0,0 +1,59 @@
+SET debug_compile=
+SET debug_link=
+
+if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+
+copy ..\..\back-ends\foma\libfoma.* .
+copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+cl %debug_compile% /EHsc /LD /Felibhfst.dll ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp /link %debug_link% openfst.lib libfoma.lib
diff --git a/scripts/make-openfstwin.bat b/scripts/make-openfstwin.bat
new file mode 100644
index 0000000..ef35795
--- /dev/null
+++ b/scripts/make-openfstwin.bat
@@ -0,0 +1,12 @@
+SET debug_compile=
+
+if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /MDd
+
+SET extra_cpp_files=
+SET extra_obj_files=
+
+if exist symbol-table-ops.cpp SET extra_cpp_files=symbol-table-ops.cpp
+if exist symbol-table-ops.cpp SET extra_obj_files=symbol-table-ops.obj
+
+cl %debug_compile% /EHsc /LD /DOPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 /Feopenfst.dll /I ..\include\ compat.cpp flags.cpp fst.cpp properties.cpp symbol-table.cpp %extra_cpp_files% util.cpp
+!! lib.exe /OUT:openfst.lib compat.obj flags.obj fst.obj properties.obj symbol-table.obj %extra_obj_files% util.obj
diff --git a/scripts/make-parsers.bat b/scripts/make-parsers.bat
new file mode 100644
index 0000000..cbf443d
--- /dev/null
+++ b/scripts/make-parsers.bat
@@ -0,0 +1 @@
+cl /c /EHsc /I. /I..\..\.. /I..\..\..\libhfst\src /I..\..\..\libhfst\src\parsers /I..\..\..\back-ends\foma /I..\..\..\back-ends /I..\..\..\back-ends\openfstwin\src\include xre_lex.cpp xre_parse.cpp pmatch_parse.cpp pmatch_lex.cpp lexc-parser.cpp lexc-lexer.cpp LexcCompiler.cpp PmatchCompiler.cpp XreCompiler.cpp lexc-utils.cpp pmatch_utils.cpp xre_utils.cpp
diff --git a/scripts/make-python-bindings.bat b/scripts/make-python-bindings.bat
new file mode 100644
index 0000000..63fed97
--- /dev/null
+++ b/scripts/make-python-bindings.bat
@@ -0,0 +1,92 @@
+ at echo OFF
+
+rem SET debug_compile=
+rem SET debug_link=
+rem if "%1%"=="/DEBUG" SET debug_compile=/Zi /O2 /bigobj /MDd
+rem if "%1%"=="/DEBUG" SET debug_link=/DEBUG /release
+rem copy ..\..\back-ends\foma\libfoma.* .
+rem copy ..\..\back-ends\openfstwin\src\lib\openfst.* .
+
+if not exist libhfst_win.i echo Error: missing file "libhfst_win.i" && ^
+exit /B
+
+if not exist C:\python33\libs\python33.lib echo Error: missing file "C:\python33\libs\python33.lib" && ^
+exit /B
+
+if not exist C:\swigwin-3.0.5\swig.exe echo Error: missing executable "C:\swigwin-3.0.5\swig.exe" && ^
+exit /B
+
+if not exist "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" echo Error: missing include directory "C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" && ^
+exit /B
+
+if exist "..\..\back-ends\foma\flags.c" echo Renaming foma's flags.c into _flags.c as there is a file with the same name in openfst ^
+move ..\..\back-ends\foma\flags.c ..\..\back-ends\foma\_flags.c
+
+ at echo ON
+
+copy C:\python33\libs\python33.lib .
+
+C:\swigwin-3.0.5\swig.exe -python -c++ -I"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A\Include" -Wall -o _libhfst.cpp libhfst_win.i
+
+cl /EHsc /LD /Fe_libhfst.pyd ^
+/D HAVE_FOMA /D HAVE_OPENFST /D HFSTEXPORT /D OPENFSTEXPORT /D_MSC_VER /DWINDOWS /DWIN32 ^
+/I..\..\libhfst\src /I..\..\back-ends\foma /I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /I ..\.. ^
+/Iparsers /I..\.. /I..\..\libhfst\src /I..\..\libhfst\src\parsers /I..\..\back-ends\foma ^
+/I..\..\back-ends /I..\..\back-ends\openfstwin\src\include /IC:\Python33\include\ ^
+HfstApply.cpp ^
+HfstInputStream.cpp ^
+HfstTransducer.cpp ^
+HfstOutputStream.cpp ^
+HfstRules.cpp ^
+HfstXeroxRules.cpp ^
+HfstDataTypes.cpp ^
+HfstSymbolDefs.cpp ^
+HfstTokenizer.cpp ^
+HfstFlagDiacritics.cpp ^
+HfstExceptionDefs.cpp ^
+HarmonizeUnknownAndIdentitySymbols.cpp ^
+HfstLookupFlagDiacritics.cpp ^
+HfstEpsilonHandler.cpp ^
+implementations\HfstTransitionGraph.cpp ^
+implementations\ConvertTransducerFormat.cpp ^
+implementations\HfstTropicalTransducerTransitionData.cpp ^
+implementations\ConvertTropicalWeightTransducer.cpp ^
+implementations\ConvertLogWeightTransducer.cpp ^
+implementations\ConvertFomaTransducer.cpp ^
+implementations\ConvertOlTransducer.cpp ^
+implementations\TropicalWeightTransducer.cpp ^
+implementations\LogWeightTransducer.cpp ^
+implementations\FomaTransducer.cpp ^
+implementations\HfstOlTransducer.cpp ^
+implementations\compose_intersect\ComposeIntersectRulePair.cpp ^
+implementations\compose_intersect\ComposeIntersectLexicon.cpp ^
+implementations\compose_intersect\ComposeIntersectRule.cpp ^
+implementations\compose_intersect\ComposeIntersectFst.cpp ^
+implementations\compose_intersect\ComposeIntersectUtilities.cpp ^
+implementations\optimized-lookup\transducer.cpp ^
+implementations\optimized-lookup\convert.cpp ^
+implementations\optimized-lookup\ospell.cpp ^
+implementations\optimized-lookup\pmatch.cpp ^
+implementations\optimized-lookup\find_epsilon_loops.cpp ^
+parsers\xre_lex.cpp ^
+parsers\xre_parse.cpp ^
+parsers\pmatch_parse.cpp ^
+parsers\pmatch_lex.cpp ^
+parsers\lexc-parser.cpp ^
+parsers\lexc-lexer.cpp ^
+parsers\LexcCompiler.cpp ^
+parsers\PmatchCompiler.cpp ^
+parsers\XreCompiler.cpp ^
+parsers\lexc-utils.cpp ^
+parsers\pmatch_utils.cpp ^
+parsers\xre_utils.cpp ^
+..\..\back-ends\openfstwin\src\lib\compat.cpp ^
+..\..\back-ends\openfstwin\src\lib\flags.cpp ^
+..\..\back-ends\openfstwin\src\lib\fst.cpp ^
+..\..\back-ends\openfstwin\src\lib\properties.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table.cpp ^
+..\..\back-ends\openfstwin\src\lib\symbol-table-ops.cpp ^
+..\..\back-ends\openfstwin\src\lib\util.cpp ^
+..\..\back-ends\foma\*.c ^
+_libhfst.cpp
+
diff --git a/scripts/package-static-binaries.sh b/scripts/package-static-binaries.sh
new file mode 100755
index 0000000..d140822
--- /dev/null
+++ b/scripts/package-static-binaries.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+if [ "$1" = "-h" ]; then
+ echo "Usage: $0 DIRNAME";
+ exit 0;
+fi
+
+if [ "$1" = "" ]; then
+ echo "Usage: $0 DIRNAME";
+ exit 1;
+fi
+
+if [ -d "$1" ]; then
+ echo "directory $1 exists"
+ exit 1;
+else
+ mkdir $1;
+fi
+
+# package all eight statically compiled tools
+cp tools/src/parsers/.libs/hfst-xfst $1/
+cp tools/src/.libs/hfst-lexc $1/
+cp tools/src/.libs/hfst-lookup $1/
+cp tools/src/.libs/hfst-optimized-lookup $1/
+cp tools/src/.libs/hfst-pmatch $1/
+cp tools/src/.libs/hfst-pmatch2fst $1/
+cp tools/src/hfst-proc/.libs/hfst-apertium-proc $1/hfst-proc
+cp tools/src/hfst-twolc/src/.libs/htwolcpre1 $1/
+cp tools/src/hfst-twolc/src/.libs/htwolcpre2 $1/
+cp tools/src/hfst-twolc/src/.libs/htwolcpre3 $1/
+cp scripts/hfst-twolc-bin $1/hfst-twolc
diff --git a/scripts/stdint.h b/scripts/stdint.h
new file mode 100644
index 0000000..59d0673
--- /dev/null
+++ b/scripts/stdint.h
@@ -0,0 +1,247 @@
+// ISO C9x compliant stdint.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124
+//
+// Copyright (c) 2006-2008 Alexander Chemeris
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. The name of the author may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_STDINT_H_ // [
+#define _MSC_STDINT_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include <limits.h>
+
+// For Visual Studio 6 in C++ mode and for many Visual Studio versions when
+// compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}'
+// or compiler give many errors like this:
+// error C2733: second C linkage of overloaded function 'wmemchr' not allowed
+#ifdef __cplusplus
+extern "C" {
+#endif
+# include <wchar.h>
+#ifdef __cplusplus
+}
+#endif
+
+// Define _W64 macros to mark types changing their size, like intptr_t.
+#ifndef _W64
+# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
+# define _W64 __w64
+# else
+# define _W64
+# endif
+#endif
+
+
+// 7.18.1 Integer types
+
+// 7.18.1.1 Exact-width integer types
+
+// Visual Studio 6 and Embedded Visual C++ 4 doesn't
+// realize that, e.g. char has the same size as __int8
+// so we give up on __intX for them.
+#if (_MSC_VER < 1300)
+ typedef signed char int8_t;
+ typedef signed short int16_t;
+ typedef signed int int32_t;
+ typedef unsigned char uint8_t;
+ typedef unsigned short uint16_t;
+ typedef unsigned int uint32_t;
+#else
+ typedef signed __int8 int8_t;
+ typedef signed __int16 int16_t;
+ typedef signed __int32 int32_t;
+ typedef unsigned __int8 uint8_t;
+ typedef unsigned __int16 uint16_t;
+ typedef unsigned __int32 uint32_t;
+#endif
+typedef signed __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+
+
+// 7.18.1.2 Minimum-width integer types
+typedef int8_t int_least8_t;
+typedef int16_t int_least16_t;
+typedef int32_t int_least32_t;
+typedef int64_t int_least64_t;
+typedef uint8_t uint_least8_t;
+typedef uint16_t uint_least16_t;
+typedef uint32_t uint_least32_t;
+typedef uint64_t uint_least64_t;
+
+// 7.18.1.3 Fastest minimum-width integer types
+typedef int8_t int_fast8_t;
+typedef int16_t int_fast16_t;
+typedef int32_t int_fast32_t;
+typedef int64_t int_fast64_t;
+typedef uint8_t uint_fast8_t;
+typedef uint16_t uint_fast16_t;
+typedef uint32_t uint_fast32_t;
+typedef uint64_t uint_fast64_t;
+
+// 7.18.1.4 Integer types capable of holding object pointers
+#ifdef _WIN64 // [
+ typedef signed __int64 intptr_t;
+ typedef unsigned __int64 uintptr_t;
+#else // _WIN64 ][
+ typedef _W64 signed int intptr_t;
+ typedef _W64 unsigned int uintptr_t;
+#endif // _WIN64 ]
+
+// 7.18.1.5 Greatest-width integer types
+typedef int64_t intmax_t;
+typedef uint64_t uintmax_t;
+
+
+// 7.18.2 Limits of specified-width integer types
+
+#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259
+
+// 7.18.2.1 Limits of exact-width integer types
+#define INT8_MIN ((int8_t)_I8_MIN)
+#define INT8_MAX _I8_MAX
+#define INT16_MIN ((int16_t)_I16_MIN)
+#define INT16_MAX _I16_MAX
+#define INT32_MIN ((int32_t)_I32_MIN)
+#define INT32_MAX _I32_MAX
+#define INT64_MIN ((int64_t)_I64_MIN)
+#define INT64_MAX _I64_MAX
+#define UINT8_MAX _UI8_MAX
+#define UINT16_MAX _UI16_MAX
+#define UINT32_MAX _UI32_MAX
+#define UINT64_MAX _UI64_MAX
+
+// 7.18.2.2 Limits of minimum-width integer types
+#define INT_LEAST8_MIN INT8_MIN
+#define INT_LEAST8_MAX INT8_MAX
+#define INT_LEAST16_MIN INT16_MIN
+#define INT_LEAST16_MAX INT16_MAX
+#define INT_LEAST32_MIN INT32_MIN
+#define INT_LEAST32_MAX INT32_MAX
+#define INT_LEAST64_MIN INT64_MIN
+#define INT_LEAST64_MAX INT64_MAX
+#define UINT_LEAST8_MAX UINT8_MAX
+#define UINT_LEAST16_MAX UINT16_MAX
+#define UINT_LEAST32_MAX UINT32_MAX
+#define UINT_LEAST64_MAX UINT64_MAX
+
+// 7.18.2.3 Limits of fastest minimum-width integer types
+#define INT_FAST8_MIN INT8_MIN
+#define INT_FAST8_MAX INT8_MAX
+#define INT_FAST16_MIN INT16_MIN
+#define INT_FAST16_MAX INT16_MAX
+#define INT_FAST32_MIN INT32_MIN
+#define INT_FAST32_MAX INT32_MAX
+#define INT_FAST64_MIN INT64_MIN
+#define INT_FAST64_MAX INT64_MAX
+#define UINT_FAST8_MAX UINT8_MAX
+#define UINT_FAST16_MAX UINT16_MAX
+#define UINT_FAST32_MAX UINT32_MAX
+#define UINT_FAST64_MAX UINT64_MAX
+
+// 7.18.2.4 Limits of integer types capable of holding object pointers
+#ifdef _WIN64 // [
+# define INTPTR_MIN INT64_MIN
+# define INTPTR_MAX INT64_MAX
+# define UINTPTR_MAX UINT64_MAX
+#else // _WIN64 ][
+# define INTPTR_MIN INT32_MIN
+# define INTPTR_MAX INT32_MAX
+# define UINTPTR_MAX UINT32_MAX
+#endif // _WIN64 ]
+
+// 7.18.2.5 Limits of greatest-width integer types
+#define INTMAX_MIN INT64_MIN
+#define INTMAX_MAX INT64_MAX
+#define UINTMAX_MAX UINT64_MAX
+
+// 7.18.3 Limits of other integer types
+
+#ifdef _WIN64 // [
+# define PTRDIFF_MIN _I64_MIN
+# define PTRDIFF_MAX _I64_MAX
+#else // _WIN64 ][
+# define PTRDIFF_MIN _I32_MIN
+# define PTRDIFF_MAX _I32_MAX
+#endif // _WIN64 ]
+
+#define SIG_ATOMIC_MIN INT_MIN
+#define SIG_ATOMIC_MAX INT_MAX
+
+#ifndef SIZE_MAX // [
+# ifdef _WIN64 // [
+# define SIZE_MAX _UI64_MAX
+# else // _WIN64 ][
+# define SIZE_MAX _UI32_MAX
+# endif // _WIN64 ]
+#endif // SIZE_MAX ]
+
+// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h>
+#ifndef WCHAR_MIN // [
+# define WCHAR_MIN 0
+#endif // WCHAR_MIN ]
+#ifndef WCHAR_MAX // [
+# define WCHAR_MAX _UI16_MAX
+#endif // WCHAR_MAX ]
+
+#define WINT_MIN 0
+#define WINT_MAX _UI16_MAX
+
+#endif // __STDC_LIMIT_MACROS ]
+
+
+// 7.18.4 Limits of other integer types
+
+#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260
+
+// 7.18.4.1 Macros for minimum-width integer constants
+
+#define INT8_C(val) val##i8
+#define INT16_C(val) val##i16
+#define INT32_C(val) val##i32
+#define INT64_C(val) val##i64
+
+#define UINT8_C(val) val##ui8
+#define UINT16_C(val) val##ui16
+#define UINT32_C(val) val##ui32
+#define UINT64_C(val) val##ui64
+
+// 7.18.4.2 Macros for greatest-width integer constants
+#define INTMAX_C INT64_C
+#define UINTMAX_C UINT64_C
+
+#endif // __STDC_CONSTANT_MACROS ]
+
+
+#endif // _MSC_STDINT_H_ ]
diff --git a/scripts/test-hfst-tools.bat b/scripts/test-hfst-tools.bat
new file mode 100644
index 0000000..6d05bad
--- /dev/null
+++ b/scripts/test-hfst-tools.bat
@@ -0,0 +1,31 @@
+ at echo off
+
+for %%P IN (hfst-xfst, hfst-lexc, hfst-lookup, hfst-optimized-lookup, hfst-pmatch2fst, hfst-pmatch, hfst-proc, hfst-twolc.bat) DO call %%P --help >NUL 2>NUL && echo %%P help message: ok && call %%P --version >NUL 2>NUL && echo %%P version message: ok
+
+ at echo on
+
+hfst-xfst --silent -F test.xfst
+
+type test_xfst_result.txt | hfst-lookup foobar.hfst --pipe-mode=input
+
+type test_xfst_result.txt | hfst-optimized-lookup -w foobar.olw --pipe-mode=input
+
+hfst-proc foobar.olw test_xfst_result.txt output_from_proc && type output_from_proc
+
+hfst-lexc test.lexc > lexc.hfst
+
+type test_lexc_result.txt | hfst-lookup lexc.hfst --pipe-mode=input
+
+call hfst-twolc.bat test.twolc > twolc.hfst
+
+type test_twolc_result.txt | hfst-lookup twolc.hfst --pipe-mode=input
+
+hfst-pmatch2fst --verbose test.pmatch > pmatch.hfst
+
+echo ^^^^
+echo test hfst-pmatch manually and copy input from test_pmatch_result.txt
+echo ^^^^
+
+
+
+
diff --git a/scripts/test-libhfst.bat b/scripts/test-libhfst.bat
new file mode 100644
index 0000000..536c83d
--- /dev/null
+++ b/scripts/test-libhfst.bat
@@ -0,0 +1,2 @@
+cl /EHsc /D_MSC_VER /DWINDOWS /DWIN32 test-libhfst.cpp /link libhfst.lib
+test-libhfst.exe
diff --git a/scripts/test-openfstwin.bat b/scripts/test-openfstwin.bat
new file mode 100644
index 0000000..c8686a4
--- /dev/null
+++ b/scripts/test-openfstwin.bat
@@ -0,0 +1,2 @@
+cl /EHsc /D_MSC_VER /DWINDOWS /DWIN32 /I..\include\ test-openfstwin.cpp /link openfst.lib
+test-openfstwin.exe
diff --git a/scripts/test_libhfst_win.py b/scripts/test_libhfst_win.py
new file mode 100644
index 0000000..271ab19
--- /dev/null
+++ b/scripts/test_libhfst_win.py
@@ -0,0 +1,56 @@
+import libhfst
+
+print('')
+print('Creating a word list foo::2, bar::3.5, baz::0')
+print('')
+tr = libhfst.word_list([
+('foo',2),
+('bar',3.5),
+('baz',)])
+print(tr)
+
+print('')
+print('Creating a word pair list foo:FOO::2, bar:BAR::3.5, baz:baz::0')
+print('')
+tr = libhfst.word_pair_list([
+('foo','FOO',2),
+('bar','BAR',3.5),
+('baz','baz')])
+print(tr)
+
+print('')
+print('Testing lookup')
+print('')
+holps = tr.lookup_fd('foo', 5)
+for path in holps:
+ print('weight\toutput')
+ print(path[0],end='')
+ print('\t',end='')
+ print(path[1])
+ print()
+
+#print('Testing libhfst module...')
+
+#import libhfst
+
+#print(libhfst.fst_type_to_string(libhfst.get_default_fst_type()))
+
+#print(libhfst.fst('foo'))
+#print(libhfst.fst('foo', 'bar'))
+#print(libhfst.regex('[f o o]:[b a r]'))
+#print(libhfst.word('foo'))
+#print(libhfst.word_pair('foo', 'bar'))
+
+#print(libhfst.word_list(['foo', 'bar', 'baz']))
+#spv = libhfst.StringPairVector((('foo', 'bar'), ('bar', 'baz'), ('foo', 'foo')))
+#print(libhfst.word_pair_list(spv))
+
+#tr1 = libhfst.HfstTransducer('foo', 'bar', libhfst.TROPICAL_OPENFST_TYPE)
+#tr2 = libhfst.HfstTransducer('bar', 'baz', libhfst.TROPICAL_OPENFST_TYPE)
+
+#tr1.concatenate(tr2)
+#print(tr1)
+#print('--')
+#print(tr2)
+
+#print('Tests passed.')
diff --git a/scripts/windows_tests/test.lexc b/scripts/windows_tests/test.lexc
new file mode 100644
index 0000000..bb07991
--- /dev/null
+++ b/scripts/windows_tests/test.lexc
@@ -0,0 +1,4 @@
+LEXICON Root
+föö # ;
+bär # ;
+BÅÅZ # ;
diff --git a/scripts/windows_tests/test.pmatch b/scripts/windows_tests/test.pmatch
new file mode 100644
index 0000000..d2aec21
--- /dev/null
+++ b/scripts/windows_tests/test.pmatch
@@ -0,0 +1,5 @@
+Define CapWord UppercaseAlpha Alpha* ;
+Define StreetWordFr [{avenué} | {boûlevard} | {ruè}] ;
+Define DeFr [ [{dë} | {dù} | {dés} | {dê là}] Whitespace ] | [{d'} | {l'}] ;
+Define StreetFr StreetWordFr (Whitespace DeFr) CapWord+ ;
+Define TOP StreetFr EndTag(PseudoFrenchStreetName) ;
diff --git a/scripts/windows_tests/test.twolc b/scripts/windows_tests/test.twolc
new file mode 100644
index 0000000..c8f94a8
--- /dev/null
+++ b/scripts/windows_tests/test.twolc
@@ -0,0 +1,4 @@
+Alphabet föö bär å ;
+Rules
+"rule name"
+föö:bär => å _ å ;
diff --git a/scripts/windows_tests/test.xfst b/scripts/windows_tests/test.xfst
new file mode 100644
index 0000000..85132d9
--- /dev/null
+++ b/scripts/windows_tests/test.xfst
@@ -0,0 +1,5 @@
+regex föö:bär::1.5;
+save stack foobar.hfst
+lookup-optimize
+save stack foobar.olw
+exit
diff --git a/scripts/windows_tests/test_lexc_result.txt b/scripts/windows_tests/test_lexc_result.txt
new file mode 100644
index 0000000..06051bb
--- /dev/null
+++ b/scripts/windows_tests/test_lexc_result.txt
@@ -0,0 +1,4 @@
+föö
+bär
+BÅÅZ
+foo
diff --git a/scripts/windows_tests/test_pmatch_result.txt b/scripts/windows_tests/test_pmatch_result.txt
new file mode 100644
index 0000000..3165435
--- /dev/null
+++ b/scripts/windows_tests/test_pmatch_result.txt
@@ -0,0 +1 @@
+Je marche seul dans l'avenué dés Ternes
diff --git a/scripts/windows_tests/test_twolc_result.txt b/scripts/windows_tests/test_twolc_result.txt
new file mode 100644
index 0000000..1737db4
--- /dev/null
+++ b/scripts/windows_tests/test_twolc_result.txt
@@ -0,0 +1,3 @@
+åfööå
+föö
+åföööå
diff --git a/scripts/windows_tests/test_xfst_result.txt b/scripts/windows_tests/test_xfst_result.txt
new file mode 100644
index 0000000..75e1f02
--- /dev/null
+++ b/scripts/windows_tests/test_xfst_result.txt
@@ -0,0 +1,3 @@
+föö
+foo
+FOO
diff --git a/swig/libhfst.i b/swig/libhfst.i
index 7e81cfd..58b86d7 100644
--- a/swig/libhfst.i
+++ b/swig/libhfst.i
@@ -363,6 +363,20 @@ public:
$self->write_in_att_format(tmp);
return tmp;
}
+
+ HfstTransducer & __add__(const HfstTransducer & another) {
+ return $self->concatenate(another);
+ }
+ HfstTransducer & __sub__(const HfstTransducer & another) {
+ return $self->subtract(another);
+ }
+ HfstTransducer & __or__(const HfstTransducer & another) {
+ return $self->intersect(another);
+ }
+ HfstTransducer & __and__(const HfstTransducer & another) {
+ return $self->disjunct(another);
+ }
+
};
};
@@ -406,7 +420,7 @@ public:
LexcCompiler(hfst::ImplementationType impl);
LexcCompiler& parse(FILE* infile);
LexcCompiler& parse(const char* filename);
- LexcCompiler& setVerbosity(bool verbose);
+ LexcCompiler& setVerbosity(unsigned int verbose);
LexcCompiler& addAlphabet(const std::string& alphabet);
LexcCompiler& setCurrentLexiconName(const std::string& lexicon_name);
LexcCompiler& addStringEntry(const std::string& entry, const std::string& continuation, const double weight);
diff --git a/swig/setup.py b/swig/setup.py
index 6ebcb85..ac83d18 100644
--- a/swig/setup.py
+++ b/swig/setup.py
@@ -26,7 +26,7 @@ libhfst_module = Extension('_libhfst',
# ["libhfst-NN.dll", "libgcc_s_seh-1.dll"] or
# ["libhfst-NN.dll", "libgcc_s_dw2-1.dll"] or
setup(name = 'libhfst_swig',
- version = '3.8.1_beta',
+ version = '3.8.2_beta',
author = 'HFST team',
author_email = 'hfst-bugs at helsinki.fi',
url = 'http://hfst.sourceforge.net',
diff --git a/test/tools/proc-functionality.sh b/test/tools/proc-functionality.sh
index 14b2c43..33922e1 100755
--- a/test/tools/proc-functionality.sh
+++ b/test/tools/proc-functionality.sh
@@ -99,7 +99,7 @@ fi
rm test.strings
## skip new test introduced in version 3014...
-exit 77
+exit 0
#if ! $TOOLDIR/hfst-proc/hfst-apertium-proc compounds2.hfstol < $srcdir/proc-compounds2.strings | tr -d '\r' > test.strings ; then
# echo compound fail
diff --git a/tools/src/HfstStrings2FstTokenizer.cc b/tools/src/HfstStrings2FstTokenizer.cc
index 5ea13ab..8ac5db8 100644
--- a/tools/src/HfstStrings2FstTokenizer.cc
+++ b/tools/src/HfstStrings2FstTokenizer.cc
@@ -2,6 +2,8 @@
#include <algorithm>
#include <cstring>
+namespace hfst {
+
HfstStrings2FstTokenizer::HfstStrings2FstTokenizer
(StringVector &multichar_symbols,const std::string &eps):
eps(eps)
@@ -93,18 +95,18 @@ StringPairVector HfstStrings2FstTokenizer::make_pair_vector
it != v.end();
++it)
{
- if (not is_pair_input_symbol(it,v.end()))
+ if (! is_pair_input_symbol(it,v.end()))
{
std::string symbol = unescape(*it);
- symbol = (symbol.empty() or symbol == eps ?
+ symbol = (symbol.empty() || symbol == eps ?
EPSILON_SYMBOL : symbol);
spv.push_back(StringPair(symbol,symbol)); }
else
{
- std::string input = (it->empty() or *it == eps ?
+ std::string input = (it->empty() || *it == eps ?
EPSILON_SYMBOL : unescape(*it));
++(++it);
- std::string output = (it->empty() or *it == eps ?
+ std::string output = (it->empty() || *it == eps ?
EPSILON_SYMBOL : unescape(*it));
spv.push_back(StringPair(input,output));
}
@@ -118,15 +120,15 @@ StringPairVector HfstStrings2FstTokenizer::make_pair_vector
StringPairVector spv;
StringVector::const_iterator input_it = input.begin();
StringVector::const_iterator output_it = output.begin();
- while (input_it != input.end() and output_it != output.end())
+ while (input_it != input.end() && output_it != output.end())
{
std::string input_symbol = unescape(*input_it);
std::string output_symbol = unescape(*output_it);
spv.push_back
- (StringPair(input_symbol.empty() or input_symbol == eps ?
+ (StringPair(input_symbol.empty() || input_symbol == eps ?
EPSILON_SYMBOL : input_symbol,
- output_symbol.empty() or output_symbol == eps ?
+ output_symbol.empty() || output_symbol == eps ?
EPSILON_SYMBOL : output_symbol));
++input_it;
++output_it;
@@ -136,14 +138,14 @@ StringPairVector HfstStrings2FstTokenizer::make_pair_vector
for ( ; output_it != output.end(); ++output_it)
{ spv.push_back
(StringPair(EPSILON_SYMBOL,
- output_it->empty() or *output_it == eps ?
+ output_it->empty() || *output_it == eps ?
EPSILON_SYMBOL : unescape(*output_it))); }
}
else
{
for ( ; input_it != input.end(); ++input_it)
{ spv.push_back
- (StringPair(input_it->empty() or *input_it == eps ?
+ (StringPair(input_it->empty() || *input_it == eps ?
EPSILON_SYMBOL : unescape(*input_it),
EPSILON_SYMBOL)); }
}
@@ -202,7 +204,7 @@ bool HfstStrings2FstTokenizer::is_pair_input_symbol
void HfstStrings2FstTokenizer::check_cols(const std::string &symbol)
{
- if (not symbol.empty())
+ if (! symbol.empty())
{
if (symbol[0] == COL_CHAR)
{ throw UnescapedColsFound(); }
@@ -211,7 +213,7 @@ void HfstStrings2FstTokenizer::check_cols(const std::string &symbol)
{
if (symbol[pos-1] != BACKSLASH_CHAR)
{ throw UnescapedColsFound(); }
- if (pos > 1 and symbol[pos-2] == BACKSLASH_CHAR)
+ if (pos > 1 && symbol[pos-2] == BACKSLASH_CHAR)
{ throw UnescapedColsFound(); }
}
}
@@ -238,10 +240,10 @@ StringVector HfstStrings2FstTokenizer::split_at_spaces(const std::string &str)
StringVector res;
for (StringVector::const_iterator it = sv.begin(); it != sv.end(); ++it)
{
- if (*it == SPACE and not symbol.empty())
+ if (*it == SPACE && ! symbol.empty())
{
res.push_back(symbol);
- while (it + 1 != sv.end() and *(it + 1) == SPACE)
+ while (it + 1 != sv.end() && *(it + 1) == SPACE)
{ ++it; }
symbol = EMPTY;
if (it == sv.end())
@@ -249,10 +251,10 @@ StringVector HfstStrings2FstTokenizer::split_at_spaces(const std::string &str)
}
else if (* it == SPACE)
{
- while (it + 1 != sv.end() and *(it + 1) == SPACE)
+ while (it + 1 != sv.end() && *(it + 1) == SPACE)
{ ++it; }
}
- else if (*it == COL and not symbol.empty())
+ else if (*it == COL && ! symbol.empty())
{
res.push_back(symbol);
res.push_back(COL);
@@ -263,14 +265,16 @@ StringVector HfstStrings2FstTokenizer::split_at_spaces(const std::string &str)
else
{ symbol += *it; }
}
- if (not symbol.empty())
+ if (! symbol.empty())
{ res.push_back(symbol); }
return res;
}
+} // namespace hfst
+
#ifdef TEST_FST_2_STRINGS_TOKENIZER
void test_ps
-(const std::string &input,HfstStrings2FstTokenizer &tokenizer,bool spaces)
+(const std::string &input,hfst::HfstStrings2FstTokenizer &tokenizer,bool spaces)
{
std::cout << "Tokenizing: " << input << std::endl;
std::cout << "Tokenized:" << std::endl;
@@ -288,7 +292,7 @@ void test_ps
}
void test_sp
-(const std::string &input,HfstStrings2FstTokenizer &tokenizer,bool spaces)
+(const std::string &input,hfst::HfstStrings2FstTokenizer &tokenizer,bool spaces)
{
std::cout << "Tokenizing: " << input << std::endl;
std::cout << "Tokenized:" << std::endl;
@@ -311,7 +315,7 @@ int main(void)
multichar_symbols.push_back("##");
multichar_symbols.push_back("+NOM");
multichar_symbols.push_back(":NOM:SG");
- HfstStrings2FstTokenizer tokenizer(multichar_symbols,"@_EPS_@");
+ hfst::HfstStrings2FstTokenizer tokenizer(multichar_symbols,"@_EPS_@");
test_ps("@_EPS_@:xa:b\\::c\\\\d:\\e:NOM:SG:\\+NOM:",tokenizer,false);
test_ps("@_EPS_@:x a:b \\::c \\\\ d:\\e \\:NOM\\:SG\\::\\+ N O M : ",
tokenizer,true);
diff --git a/tools/src/HfstStrings2FstTokenizer.h b/tools/src/HfstStrings2FstTokenizer.h
index af3c2fd..497c47a 100644
--- a/tools/src/HfstStrings2FstTokenizer.h
+++ b/tools/src/HfstStrings2FstTokenizer.h
@@ -29,8 +29,6 @@ typedef std::vector<std::string> StringVector;
typedef std::pair<std::string,std::string> StringPair;
typedef std::vector<StringPair> StringPairVector;
-using hfst::HfstTokenizer;
-
#define COL ":"
#define BACKSLASH "\\"
#define SPACE " "
@@ -45,6 +43,8 @@ using hfst::HfstTokenizer;
#define TAB_ESCAPE "@_TAB_@"
#define SPACE_ESCAPE "@_SPACE_@"
+namespace hfst {
+
class EmptyMulticharSymbol
{};
@@ -69,7 +69,7 @@ class HfstStrings2FstTokenizer
protected:
- HfstTokenizer tokenizer;
+ hfst::HfstTokenizer tokenizer;
std::string eps;
// Add the multichar symbol XYZ to the tokenizer.
@@ -105,4 +105,6 @@ class HfstStrings2FstTokenizer
void check_cols(const std::string &symbol);
};
+} // namespace hfst
+
#endif // HEADER_STRINGS_2_FST_TOKENIZER_H
diff --git a/tools/src/Makefile.am b/tools/src/Makefile.am
index 9f03e80..226d936 100644
--- a/tools/src/Makefile.am
+++ b/tools/src/Makefile.am
@@ -95,7 +95,7 @@ endif
if WANT_INVERT
MAYBE_INVERT=hfst-invert$(EXEEXT)
endif
-if WANT_LEXC_WRAPPER
+if GENERATE_LEXC_WRAPPER
MAYBE_LEXC_WRAPPER=hfst-lexc-wrapper$(EXEEXT)
endif
if WANT_LEXC
@@ -266,7 +266,7 @@ hfst_lexc_SOURCES=hfst-lexc-compiler.cc $(HFST_COMMON_SRC)
# parsers/LexcCompiler.h parsers/lexc-utils.h
#hfst_lexc2fst_BUILT=parsers/lexc-parser.h parsers/xre_parse.h
hfst_lookup_SOURCES=hfst-lookup.cc $(HFST_COMMON_SRC) \
-HfstStrings2FstTokenizer.cc
+HfstStrings2FstTokenizer.cc hfst-string-conversions.cc
hfst_pair_test_SOURCES=hfst-pair-test.cc $(HFST_COMMON_SRC) \
HfstStrings2FstTokenizer.cc
hfst_minimize_SOURCES=hfst-minimize.cc $(HFST_COMMON_SRC)
diff --git a/tools/src/hfst-commandline.cc b/tools/src/hfst-commandline.cc
index f3f9b41..b92f12a 100644
--- a/tools/src/hfst-commandline.cc
+++ b/tools/src/hfst-commandline.cc
@@ -19,6 +19,9 @@
#if HAVE_CONFIG_H
# include <config.h>
+#else
+# define PACKAGE_STRING ""
+# define PACKAGE_BUGREPORT ""
#endif
#include <cassert>
@@ -28,7 +31,13 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <unistd.h>
+
+#ifndef _MSC_VER
+# include <unistd.h>
+#else
+# include <io.h>
+#endif
+
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
@@ -52,6 +61,12 @@
#include "HfstTransducer.h"
#include "HfstInputStream.h"
+// LLONG_MAX for 64-bit...
+#ifdef _MSC_VER
+# define strcasecmp _stricmp
+# define SSIZE_MAX LLONG_MAX
+#endif
+
#ifndef HAVE_ERROR_AT_LINE
void error_at_line(int status, int errnum, const char* filename,
unsigned int linenum, const char* fmt, ...)
@@ -209,12 +224,16 @@ hfst_strtonumber(const char *s, bool *infinite)
double rv = strtod(s, &endptr);
if (*endptr == '\0')
{
+#ifndef _MSC_VER
if (std::isinf(rv) && infinite != NULL)
{
*infinite = true;
return std::signbit(rv);
}
else if (rv > INT_MAX)
+#else
+ if (rv > INT_MAX)
+#endif
{
return INT_MAX;
}
@@ -295,6 +314,10 @@ hfst_parse_format_name(const char* s)
{
rv = hfst::FOMA_TYPE;
}
+ else if (strcasecmp(s, "xfsm") == 0)
+ {
+ rv = hfst::XFSM_TYPE;
+ }
else if ((strcasecmp(s, "optimized-lookup-unweighted") == 0) ||
(strcasecmp(s, "olu") == 0))
{
@@ -339,6 +362,9 @@ hfst_strformat(hfst::ImplementationType format)
case hfst::FOMA_TYPE:
return strdup("foma");
break;
+ case hfst::XFSM_TYPE:
+ return strdup("xfsm");
+ break;
case hfst::HFST_OL_TYPE:
return strdup("Hfst's lookup optimized, unweighted");
break;
@@ -502,9 +528,10 @@ hfst_write(int fd, const void* buf, size_t count)
int
hfst_mkstemp(char* templ)
{
-#ifdef WINDOWS
+#ifdef _WIN32
error(EXIT_FAILURE, errno,
"'int hfst_mkstemp(char * temp1)' not implemented for windows");
+ return 1; // keep compiler happy
#else
errno = 0;
int rv = mkstemp(templ);
diff --git a/tools/src/hfst-commandline.h b/tools/src/hfst-commandline.h
index 995d05c..39ad821 100644
--- a/tools/src/hfst-commandline.h
+++ b/tools/src/hfst-commandline.h
@@ -35,6 +35,12 @@
# include <error.h>
#endif
+#ifdef _MSC_VER
+# include <BaseTsd.h>
+ typedef SSIZE_T ssize_t;
+#endif
+
+
#include "HfstDataTypes.h"
namespace hfst { class HfstInputStream; } ;
diff --git a/tools/src/hfst-compare.cc b/tools/src/hfst-compare.cc
index b6480eb..9f22cc9 100644
--- a/tools/src/hfst-compare.cc
+++ b/tools/src/hfst-compare.cc
@@ -32,7 +32,12 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
#include "HfstTransducer.h"
#include "HfstInputStream.h"
@@ -177,12 +182,12 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
{
if (transducer_n_first == 1)
{
- if (not silent)
+ if (! silent)
fprintf(outfile, "%s == %s\n", firstname, secondname);
}
else
{
- if (not silent)
+ if (! silent)
fprintf(outfile, "%s[" SIZE_T_SPECIFIER "] == %s[" SIZE_T_SPECIFIER "]\n",
firstname, transducer_n_first,
secondname, transducer_n_second);
@@ -192,12 +197,12 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
{
if (transducer_n_first == 1)
{
- if (not silent)
+ if (! silent)
fprintf(outfile, "%s != %s\n", firstname, secondname);
}
else
{
- if (not silent)
+ if (! silent)
fprintf(outfile, "%s[" SIZE_T_SPECIFIER "] != %s[" SIZE_T_SPECIFIER "]\n",
firstname, transducer_n_first,
secondname, transducer_n_second);
@@ -222,7 +227,7 @@ compare_streams(HfstInputStream& firststream, HfstInputStream& secondstream)
first=0;
// delete the transducer of second stream, unless we continue reading
// the first stream and there is only one transducer in the second stream
- if ((continueReading && secondstream.is_good()) || not continueReading)
+ if ((continueReading && secondstream.is_good()) || ! continueReading)
{
delete second;
second=0;
diff --git a/tools/src/hfst-compose-intersect.cc b/tools/src/hfst-compose-intersect.cc
index 291050d..2a654ea 100644
--- a/tools/src/hfst-compose-intersect.cc
+++ b/tools/src/hfst-compose-intersect.cc
@@ -57,8 +57,9 @@ using hfst::HfstTransducerVector;
// the lexicon. Otherwise the lexicon is composed with the
// intersection of the rules.
static bool invert=false;
-
static bool encode_weights=false;
+static bool fast_ci=false;
+static bool harmonize=false;
void
print_usage()
@@ -74,8 +75,11 @@ print_usage()
" rules with the lexicon instead\n"
" of composing the lexicon with\n"
" the intersection of the rules.\n"
+ " -f, --fast Faster compose instersect using\n"
+ " more memory.\n"
" -e, --encode-weights Encode weights when minimizing\n"
" (default is false).\n"
+ " -a, --harmonize Harmonize symbols.\n"
);
//print_common_binary_program_parameter_instructions(message_out);
fprintf(message_out,
@@ -110,11 +114,13 @@ parse_options(int argc, char** argv)
HFST_GETOPT_BINARY_LONG,
{"invert", no_argument, 0, 'I'},
{"encode-weights", no_argument, 0, 'e'},
+ {"fast", no_argument, 0, 'f'},
+ {"harmonize", no_argument, 0, 'a'},
{0,0,0,0}
};
int option_index = 0;
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_BINARY_SHORT "FIe",
+ HFST_GETOPT_BINARY_SHORT "FIeHfa",
long_options, &option_index);
if (-1 == c)
{
@@ -131,6 +137,12 @@ parse_options(int argc, char** argv)
case 'e':
encode_weights = true;
break;
+ case 'f':
+ fast_ci = true;
+ break;
+ case 'a':
+ harmonize = true;
+ break;
}
}
@@ -234,6 +246,15 @@ std::string check_multi_char_symbols
return "";
}
+void harmonize_rules(HfstTransducer & lexicon, std::vector<HfstTransducer> & rules)
+{
+ for (std::vector<HfstTransducer>::iterator it = rules.begin(); it != rules.end(); it++)
+ {
+ it->harmonize(lexicon);
+ }
+ return;
+}
+
int
compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
HfstOutputStream& outstream)
@@ -306,7 +327,7 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
warning(0, 0,
"\nFound output symbols (e.g. \"%s\") in transducer in\n"
"file %s which will be filtered out because they are\n"
- "not found on the output tapes of transducers in file\n"
+ "not found on the input tapes of transducers in file\n"
"%s.",
symbol.c_str(), firstfilename, secondfilename);
}
@@ -315,12 +336,44 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
warning(0, 0,
"\nFound output multi-char symbols (\"%s\") in \n"
"transducer in file %s which are not found on the\n"
- "output tape of transducers in file %s.",
+ "input tapes of transducers in file %s.",
symbol.c_str(), firstfilename, secondfilename);
}
}
- lexicon.compose_intersect(rules,invert);
+ if (harmonize)
+ {
+ harmonize_rules(lexicon, rules);
+ }
+
+ if (fast_ci)
+ {
+ // To hopefully speed up stuff: Compose intersect the output
+ // of the lexicon with the rules and then compose the original
+ // lexicon with the result.
+
+ if (invert)
+ {
+ HfstTransducer lexicon_input(lexicon);
+ lexicon_input.input_project().minimize();
+ lexicon_input.compose_intersect(rules,true);
+
+ lexicon_input.compose(lexicon);
+ lexicon = lexicon_input;
+ }
+ else
+ {
+ HfstTransducer lexicon_output(lexicon);
+ lexicon_output.output_project().minimize();
+ lexicon_output.compose_intersect(rules,false);
+ lexicon.compose(lexicon_output);
+ }
+ }
+ else
+ {
+ lexicon.compose_intersect(rules,invert);
+ }
+
char* composed_name = static_cast<char*>(malloc(sizeof(char) *
(strlen(lexiconname) +
strlen(secondfilename) +
diff --git a/tools/src/hfst-compose.cc b/tools/src/hfst-compose.cc
index 86c7208..bcb0573 100644
--- a/tools/src/hfst-compose.cc
+++ b/tools/src/hfst-compose.cc
@@ -62,12 +62,17 @@ print_usage()
print_common_program_options(message_out);
print_common_binary_program_options(message_out);
fprintf(message_out,
+"Composition options:\n"
+" -x, --xerox-composition=VALUE Whether flag diacritics are treated as ordinary\n"
+" symbols in composition (default is false).\n"
+" -X, --xfst=VARIABLE Toggle xfst compatibility option VARIABLE.\n"
"Harmonization:\n"
" -H, --do-not-harmonize Do not harmonize symbols.\n"
" -F, --harmonize-flags Harmonize flag diacritics.\n");
fprintf(message_out, "\n");
print_common_binary_program_parameter_instructions(message_out);
fprintf(message_out, "\n");
+ fprintf(message_out, "Xfst variables are {flag-is-epsilon (default OFF)}.\n");
fprintf(message_out,
"\n"
"Examples:\n"
@@ -93,11 +98,13 @@ parse_options(int argc, char** argv)
HFST_GETOPT_BINARY_LONG,
{"harmonize-flags", no_argument, 0, 'F'},
{"do-not-harmonize", no_argument, 0, 'H'},
+ {"xerox-composition", required_argument, 0, 'x'},
+ {"xfst", required_argument, 0, 'X'},
{0,0,0,0}
};
int option_index = 0;
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_BINARY_SHORT "FH",
+ HFST_GETOPT_BINARY_SHORT "FHx:X:",
long_options, &option_index);
if (-1 == c)
{
@@ -113,6 +120,42 @@ parse_options(int argc, char** argv)
case 'H':
harmonize=false;
break;
+ case 'x':
+ {
+ const char * argument = hfst_strdup(optarg);
+ if (strcmp(argument, "yes") == 0 ||
+ strcmp(argument, "true") == 0 ||
+ strcmp(argument, "ON") == 0)
+ {
+ hfst::set_xerox_composition(true);
+ }
+ else if (strcmp(argument, "no") == 0 ||
+ strcmp(argument, "false") == 0 ||
+ strcmp(argument, "OFF") == 0)
+ {
+ hfst::set_xerox_composition(false);
+ }
+ else
+ {
+ fprintf(stderr, "Error: unknown option to --xerox-composition: '%s'\n", optarg);
+ return EXIT_FAILURE;
+ }
+ }
+ break;
+ case 'X':
+ {
+ const char * argument = hfst_strdup(optarg);
+ if (strcmp(argument, "flag-is-epsilon") == 0)
+ {
+ hfst::set_flag_is_epsilon_in_composition(true);
+ }
+ else
+ {
+ fprintf(stderr, "Error: unknown option to --xfst: '%s'\n", optarg);
+ return EXIT_FAILURE;
+ }
+ }
+ break;
#include "inc/getopt-cases-error.h"
}
}
@@ -257,6 +300,7 @@ compose_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
firststream.close();
secondstream.close();
+ outstream.flush();
outstream.close();
return EXIT_SUCCESS;
diff --git a/tools/src/hfst-concatenate.cc b/tools/src/hfst-concatenate.cc
index 64d6d66..4b424da 100644
--- a/tools/src/hfst-concatenate.cc
+++ b/tools/src/hfst-concatenate.cc
@@ -231,6 +231,7 @@ concatenate_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
firststream.close();
secondstream.close();
+ outstream.flush();
outstream.close();
return EXIT_SUCCESS;
}
diff --git a/tools/src/hfst-conjunct.cc b/tools/src/hfst-conjunct.cc
index 2125df8..b5b9c97 100644
--- a/tools/src/hfst-conjunct.cc
+++ b/tools/src/hfst-conjunct.cc
@@ -227,6 +227,7 @@ conjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
}
firststream.close();
secondstream.close();
+ outstream.flush();
outstream.close();
return EXIT_SUCCESS;
}
diff --git a/tools/src/hfst-disjunct.cc b/tools/src/hfst-disjunct.cc
index fd2c285..6f5943f 100644
--- a/tools/src/hfst-disjunct.cc
+++ b/tools/src/hfst-disjunct.cc
@@ -186,6 +186,7 @@ disjunct_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
second=0;
}
+ outstream.flush();
free(firstname);
free(secondname);
}
diff --git a/tools/src/hfst-fst2fst.cc b/tools/src/hfst-fst2fst.cc
index ae386ef..d0e4d3b 100644
--- a/tools/src/hfst-fst2fst.cc
+++ b/tools/src/hfst-fst2fst.cc
@@ -70,6 +70,7 @@ print_usage()
" -b, --use-backend-format Write result in implementation format, without any HFST wrappers\n"
" -S, --sfst Write output in (HFST's) SFST implementation\n"
" -F, --foma Write output in (HFST's) foma implementation\n"
+ " -x, --xfsm Write output in native xfsm format\n"
" -t, --openfst-tropical Write output in (HFST's) tropical weight (OpenFST) implementation\n"
" -l, --openfst-log Write output in (HFST's) log weight (OpenFST) implementation\n"
" -O, --optimized-lookup-unweighted Write output in the HFST optimized-lookup implementation\n"
@@ -79,8 +80,9 @@ print_usage()
print_common_unary_program_parameter_instructions(message_out);
fprintf(message_out,
"FMT must be name of a format usable by libhfst, i.e. one of the following:\n"
- "{ foma, openfst-tropical, openfst-log, sfst,\n"
- " optimized-lookup-weighted, optimized-lookup-unweighted }.\n");
+ "{ foma, openfst-tropical, openfst-log, sfst, xfsm\n"
+ " optimized-lookup-weighted, optimized-lookup-unweighted }.\n"
+ "Note that xfsm format is always written in native format without HFST wrappers.\n");
fprintf(message_out, "\n");
print_report_bugs();
fprintf(message_out, "\n");
@@ -103,6 +105,7 @@ parse_options(int argc, char** argv)
{"format", required_argument, 0, 'f'},
{"sfst", no_argument, 0, 'S'},
{"foma", no_argument, 0, 'F'},
+ {"xfsm", no_argument, 0, 'x'},
{"openfst-tropical", no_argument, 0, 't'},
{"openfst-log", no_argument, 0, 'l'},
{"optimized-lookup-unweighted", no_argument, 0, 'O'},
@@ -113,7 +116,7 @@ parse_options(int argc, char** argv)
int option_index = 0;
// add tool-specific options here
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "SFtlOwQf:b",
+ HFST_GETOPT_UNARY_SHORT "SFtlOwQf:bx",
long_options, &option_index);
if (-1 == c)
{
@@ -138,6 +141,9 @@ parse_options(int argc, char** argv)
case 'F':
output_type = hfst::FOMA_TYPE;
break;
+ case 'x':
+ output_type = hfst::XFSM_TYPE;
+ break;
case 't':
output_type = hfst::TROPICAL_OPENFST_TYPE;
break;
@@ -161,7 +167,7 @@ parse_options(int argc, char** argv)
{
error(EXIT_FAILURE, 0,
"You must specify an output type "
- "(one of -S, -f, -t, -l, -O, or -w)");
+ "(one of -S, -F, -t, -x, -l, -O, or -w)");
}
#include "inc/check-params-common.h"
@@ -197,6 +203,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
outstream << orig;
free(inputname);
}
+ outstream.flush(); // needed for xfsm transducers whose writing is delayed
instream.close();
outstream.close();
return EXIT_SUCCESS;
@@ -226,7 +233,7 @@ int main( int argc, char **argv ) {
verbose_printf("Reading from %s, writing to %s\n",
inputfilename, outfilename);
char* format_description = hfst_strformat(output_type);
- if (hfst_format)
+ if (hfst_format && (output_type != hfst::XFSM_TYPE))
{
verbose_printf("Writing %s format transducers with HFST3 headers\n",
format_description);
@@ -237,6 +244,16 @@ int main( int argc, char **argv ) {
" headers\n", format_description);
}
free(format_description);
+
+ if (output_type == hfst::XFSM_TYPE)
+ {
+ if (strcmp(outfilename, "<stdout>") == 0) {
+ error(EXIT_FAILURE, 0, "Writing to standard output not supported for xfsm transducers,\n"
+ "use 'hfst-fst2fst [--output|-o] OUTFILE' instead");
+ return EXIT_FAILURE;
+ }
+ }
+
// here starts the buffer handling part
HfstInputStream* instream = NULL;
try {
@@ -245,8 +262,15 @@ int main( int argc, char **argv ) {
} catch(const HfstException e) {
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
inputfilename);
+#if HAVE_XFSM
+ if (inputfile == stdin) {
+ error(EXIT_FAILURE, 0, "note that you cannot read xfsm transducers from standard input,\n"
+ "use hfst-fst2fst [--input|-i] INFILE instead");
+ }
+#endif
return EXIT_FAILURE;
}
+
HfstOutputStream* outstream = (outfile != stdout) ?
new HfstOutputStream(outfilename, output_type, hfst_format) :
new HfstOutputStream(output_type, hfst_format);
diff --git a/tools/src/hfst-fst2strings.cc b/tools/src/hfst-fst2strings.cc
index b2a9ae3..8995270 100644
--- a/tools/src/hfst-fst2strings.cc
+++ b/tools/src/hfst-fst2strings.cc
@@ -62,6 +62,8 @@ static int max_strings = 0;
static int cycles = -1;
static int nbest_strings=-1;
static int max_random_strings=-1;
+static float max_weight=-1;
+static float beam=-1;
static bool display_weights=false;
static bool eval_fd=false;
static bool filter_fd=true;
@@ -97,6 +99,8 @@ print_usage()
" -e, --epsilon-format=EPS print epsilon as EPS\n"
" -X, --xfst=VARIABLE toggle xfst compatibility option VARIABLE\n");
fprintf(message_out, "Ignore paths if:\n"
+" -b, --beam=B output string weight not within B from the weight\n"
+" of the best output string\n"
" -l, --max-in-length=MIL input string longer than MIL\n"
" -L, --max-out-length=MOL output string longer than MOL\n"
" -p, --in-prefix=OPREFIX input string not beginning with IPREFIX\n"
@@ -112,6 +116,7 @@ print_usage()
"NSTR, NBEST and NCYC default to infinity.\n"
"NBEST overrides NSTR and NCYC\n"
"NRAND overrides NBEST, NSTR and NCYC\n"
+ "B must be a non-negative float\n"
"If EPS is not given, default is empty string.\n"
"Numeric options are parsed with strtod(3).\n"
"Xfst variables supported are { obey-flags, print-flags,\n"
@@ -144,6 +149,7 @@ parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
HFST_GETOPT_UNARY_LONG,
+ {"beam", required_argument, 0, 'b'},
{"cycles", required_argument, 0, 'c'},
{"epsilon-format", required_argument, 0, 'e'},
{"in-exclude", required_argument, 0, 'u'},
@@ -163,7 +169,7 @@ parse_options(int argc, char** argv)
int option_index = 0;
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
HFST_GETOPT_UNARY_SHORT
- "Swc:e:u:p:l:L:n:r:N:U:P:X:",
+ "Swb:c:e:u:p:l:L:n:r:N:U:P:X:",
long_options, &option_index);
if (-1 == c)
{
@@ -183,6 +189,14 @@ parse_options(int argc, char** argv)
case 'r':
max_random_strings = hfst_strtoul(optarg, 10);
break;
+ case 'b':
+ beam = atof(optarg);
+ if (beam < 0)
+ {
+ std::cerr << "Invalid argument for --beam\n";
+ return EXIT_FAILURE;
+ }
+ break;
case 'c':
cycles = hfst_strtoul(optarg, 10);
break;
@@ -302,7 +316,7 @@ public:
istring.append(it->first);
ostring.append(it->second);
}
-
+ float weight = path.first;
if ((max_input_length > 0) &&
(istring.length() > max_input_length))
@@ -352,6 +366,11 @@ public:
return RetVal(true, false);
// continue searching, break off this path
}
+ if (max_weight >= 0 && weight > (max_weight + beam))
+ {
+ return RetVal(true, false);
+ // continue searching, break off this path
+ }
// the path passed the checks. Print it if it is final
if (final)
{
@@ -470,6 +489,41 @@ process_stream(HfstInputStream& instream, std::ostream& outstream)
if(input_prefix != "")
verbose_printf("input_prefix: '%s'\n", input_prefix.c_str());
+ if(beam >= 0)
+ {
+ verbose_printf("Finding the weight of the best path...\n");
+ try
+ {
+ HfstTransducer tc(t);
+ tc.n_best(1);
+ HfstTwoLevelPaths best_paths;
+ tc.extract_paths(best_paths);
+ if (best_paths.size() != 1)
+ {
+ error(EXIT_FAILURE, 0, "n_best(1) produced more than one path");
+ }
+ max_weight = best_paths.begin()->first;
+ }
+ catch (const FunctionNotImplementedException & e)
+ {
+ if (instream.get_type() == hfst::HFST_OL_TYPE ||
+ instream.get_type() == hfst::HFST_OLW_TYPE)
+ {
+ error(EXIT_FAILURE, 0, "option --beam not implemented for optimized lookup format");
+ }
+ else
+ {
+ error(EXIT_FAILURE, 0, "option --beam not implemented");
+ }
+ return EXIT_FAILURE;
+ }
+ catch(const HfstFatalException & e)
+ {
+ error(EXIT_FAILURE, 0, "n_best runs out of memory");
+ return EXIT_FAILURE;
+ }
+ }
+
if(nbest_strings > 0)
{
verbose_printf("Pruning transducer to %i best path(s)...\n",
diff --git a/tools/src/hfst-fst2txt.cc b/tools/src/hfst-fst2txt.cc
index 0b81a29..ed0b537 100644
--- a/tools/src/hfst-fst2txt.cc
+++ b/tools/src/hfst-fst2txt.cc
@@ -83,7 +83,7 @@ print_usage()
" -D, --do-not-print-weights If weights are not printed in any "
"case\n"
" -f, --format=TFMT Print output in TFMT format "
- "[default=att]\n");
+ "[default=att]\n");
fprintf(message_out, "\n");
fprintf(message_out,
"If OUTFILE or INFILE is missing or -, "
@@ -524,6 +524,11 @@ process_stream(HfstInputStream& instream, FILE* outf)
}
else
{
+ if (instream.get_type() == hfst::XFSM_TYPE) {
+ error(EXIT_FAILURE, 0, "Writing more than one transducer in text format to file not supported for xfsm transducers,\n"
+ "use [hfst-head|hfst-tail|hfst-split] to extract individual transducers from input");
+ return EXIT_FAILURE;
+ }
verbose_printf("Converting %s..." SIZE_T_SPECIFIER "\n", inputname,
transducer_n);
}
@@ -538,7 +543,7 @@ process_stream(HfstInputStream& instream, FILE* outf)
printw=true;
else if (do_not_print_weights)
printw=false;
- else if ( (type == hfst::SFST_TYPE || type == hfst::FOMA_TYPE) )
+ else if ( (type == hfst::SFST_TYPE || type == hfst::FOMA_TYPE || type == hfst::XFSM_TYPE) )
printw = false;
else if ( (type == hfst::TROPICAL_OPENFST_TYPE || type == hfst::LOG_OPENFST_TYPE) )
printw = true;
@@ -547,24 +552,52 @@ process_stream(HfstInputStream& instream, FILE* outf)
switch (format)
{
case ATT_TEXT:
- if (use_numbers)
+ if (use_numbers) // xfsm case checked earlier
t->write_in_att_format_number(outf,printw);
- else
+ else { // xfsm not yet supported
+ //if (type == hfst::XFSM_TYPE) // weights are never printed
+ // t->write_xfsm_transducer_in_att_format(outfilename);
+ //else
t->write_in_att_format(outf,printw);
+ }
break;
- case DOT_TEXT:
+ case DOT_TEXT: // xfsm case checked earlier
print_dot(outf, *t);
break;
- case PCKIMMO_TEXT:
+ case PCKIMMO_TEXT: // xfsm case checked earlier
print_pckimmo(outf, *t);
break;
case PROLOG_TEXT:
{
- HfstBasicTransducer fsm(*t);
- std::string namestr = t->get_name();
- if (namestr == "")
- namestr = "NO_NAME";
- fsm.write_in_prolog_format(outf,namestr,printw);
+ try
+ {
+ if (type == hfst::XFSM_TYPE) {
+ t->write_xfsm_transducer_in_prolog_format(outfilename); // no name or weights printed
+ }
+ else {
+ std::string namestr = t->get_name();
+ std::ostringstream ostr;
+ ostr << transducer_n;
+ std::string alt_namestr = "NO_NAME_" + ostr.str();
+
+ if (namestr == "") {
+ namestr = alt_namestr;
+ if (!silent) {
+ fprintf(stderr, "Transducer has no name, giving it a name '%s'...\n", namestr.c_str()); }
+ }
+ else {
+ namestr = alt_namestr;
+ if (!silent) {
+ fprintf(stderr, "Renaming transducer into '%s'...\n", namestr.c_str()); }
+ }
+
+ t->write_in_prolog_format(outf,namestr,printw);
+ }
+ }
+ catch (const HfstException & e)
+ {
+ error(EXIT_FAILURE, 0, "Error encountered when writing in prolog format: %s", e.name.c_str());
+ }
break;
}
default:
@@ -611,6 +644,36 @@ int main( int argc, char **argv )
inputfilename);
return EXIT_FAILURE;
}
+
+ if (instream->get_type() == hfst::XFSM_TYPE)
+ {
+ if (format == DOT_TEXT) {
+ error(EXIT_FAILURE, 0, "Output format 'dot' not supported for xfsm transducers, use 'prolog'");
+ return EXIT_FAILURE;
+ }
+ if (format == PCKIMMO_TEXT) {
+ error(EXIT_FAILURE, 0, "Output format 'pckimmo' not supported for xfsm transducers, use 'prolog'");
+ return EXIT_FAILURE;
+ }
+ if (format == ATT_TEXT) {
+ error(EXIT_FAILURE, 0, "Output format 'att' not supported for xfsm transducers, use 'prolog'");
+ return EXIT_FAILURE;
+ }
+ if (use_numbers) {
+ error(EXIT_FAILURE, 0, "Option '--use-numbers' not supported for xfsm transducers");
+ return EXIT_FAILURE;
+ }
+ if (strcmp(inputfilename, "<stdin>") == 0) {
+ error(EXIT_FAILURE, 0, "Reading from standard input not supported for xfsm transducers,\n"
+ "use 'hfst-fst2txt [--input|-i] INFILE' instead");
+ return EXIT_FAILURE;
+ }
+ if (strcmp(outfilename, "<stdout>") == 0) {
+ error(EXIT_FAILURE, 0, "Writing to standard output not supported for xfsm transducers,\n"
+ "use 'hfst-fst2txt [--output|-o] OUTFILE' instead");
+ return EXIT_FAILURE;
+ }
+ }
retval = process_stream(*instream, outfile);
diff --git a/tools/src/hfst-getopt.cc b/tools/src/hfst-getopt.cc
new file mode 100644
index 0000000..ce1e487
--- /dev/null
+++ b/tools/src/hfst-getopt.cc
@@ -0,0 +1,182 @@
+#include <cstring>
+#include <cstdio>
+#include <vector>
+
+#define no_argument 0
+#define required_argument 1
+#define optional_argument 2
+
+struct option
+{
+ const char *name;
+ int has_arg;
+ int *flag;
+ int val;
+};
+
+char * optarg = 0;
+int optopt = 0;
+int optind = 1;
+
+std::vector<char *> free_arguments;
+std::vector<char *> other_arguments;
+
+char getopt_long(int argc, char * /*const*/ argv [], const char * optstring,
+ const struct option * longopts, int * longindex)
+{
+ // check that there are more args
+ if (optind > (argc-1))
+ {
+ unsigned int i=1;
+ for (std::vector<char *>::const_iterator it = other_arguments.begin();
+ it != other_arguments.end(); it++)
+ {
+ argv[i] = *it;
+ i++;
+ }
+ optind = (int)i;
+ for (std::vector<char *>::const_iterator it = free_arguments.begin();
+ it != free_arguments.end(); it++)
+ {
+ argv[i] = *it;
+ i++;
+ }
+ return -1;
+ }
+
+ // skip free arguments
+ while(*(argv[optind]) != '-')
+ {
+ free_arguments.push_back(argv[optind]);
+ optind++;
+ if (optind > (argc-1))
+ {
+ unsigned int i=1;
+ for (std::vector<char *>::const_iterator it = other_arguments.begin();
+ it != other_arguments.end(); it++)
+ {
+ argv[i] = *it;
+ i++;
+ }
+ optind = (int)i;
+ for (std::vector<char *>::const_iterator it = free_arguments.begin();
+ it != free_arguments.end(); it++)
+ {
+ argv[i] = *it;
+ i++;
+ }
+ return -1;
+ }
+ }
+
+ other_arguments.push_back(argv[optind]);
+
+ // strdup because we are possibly modifying the argument
+ char * arg = strdup(argv[optind]); // free() should be called at the end...
+
+ // skip initial '-' signs
+ while(*arg == '-')
+ arg++;
+
+ // empty arg string
+ if (*arg == '\0')
+ {
+ optopt = -2;
+ return '?';
+ }
+
+ // whether arg is used in its short form: -f(=bar)
+ bool short_option = false;
+ arg++;
+ if (*arg == '\0' || *arg == '=')
+ short_option = true;
+ arg--;
+
+ // whether option argument is given after an '=' sign (--foo=bar, -f=bar)
+ bool eq_used = false;
+ char * argptr = arg; // this will point to the char after the '=' sign if eq_used is true
+ while (*argptr != '\0')
+ {
+ if (*argptr == '=')
+ {
+ *argptr = '\0'; // change '=' into '\0' to make string comparison easier
+ argptr++;
+ eq_used = true;
+ break;
+ }
+ argptr++;
+ }
+
+ // Go through all possible option strings
+ while(longopts->name != 0)
+ {
+ // match found, short or long format
+ if (strcmp(longopts->name, arg) == 0 || (short_option && longopts->val == (int)*arg))
+ {
+ optind++;
+ // no argument
+ if (longopts->has_arg == no_argument)
+ {
+ // argument given for an option that does not take one
+ if (eq_used)
+ {
+ fprintf(stderr, "warning: argument ignored for option '--%s'\n", longopts->name);
+ }
+ return longopts->val;
+ }
+ // required argument
+ else if (longopts->has_arg == required_argument || longopts->has_arg == optional_argument)
+ {
+ if (eq_used)
+ {
+ // we already have a pointer to the argument
+ optarg = strdup(argptr);
+ argptr--;
+ *argptr = '='; // change '\0' back to '=' (not sure if this is needed...)
+ return longopts->val;
+ }
+ // no more args, argument thus missing
+ if (optind > (argc-1))
+ {
+ if (longopts->has_arg == required_argument)
+ {
+ optopt = longopts->val;
+ return ':';
+ }
+ else
+ {
+ optopt = NULL;
+ return longopts->val;
+ }
+ }
+ // next arg is required argument (cannot be optional argument)
+ if (longopts->has_arg == required_argument)
+ {
+ optarg = strdup(argv[optind]);
+ other_arguments.push_back(argv[optind]);
+ optind++;
+ return longopts->val;
+ }
+ else
+ {
+ optopt = NULL;
+ return longopts->val;
+ }
+ }
+ // this should not happen
+ else
+ {
+ return 0;
+ }
+ }
+ longopts++;
+ }
+
+ // no match found
+ optind++;
+ optopt = -2;
+ if (short_option)
+ optopt = (int)*arg;
+ return '?';
+}
+
diff --git a/tools/src/hfst-getopt.h b/tools/src/hfst-getopt.h
new file mode 100644
index 0000000..eceb9fd
--- /dev/null
+++ b/tools/src/hfst-getopt.h
@@ -0,0 +1,18 @@
+#define no_argument 0
+#define required_argument 1
+#define optional_argument 2
+
+struct option
+{
+ const char *name;
+ int has_arg;
+ int *flag;
+ int val;
+};
+
+extern char * optarg;
+extern int optopt;
+extern int optind;
+
+char getopt_long(int argc, char * /*const*/ argv [], const char * optstring,
+ const struct option * longopts, int * longindex);
diff --git a/tools/src/hfst-head.cc b/tools/src/hfst-head.cc
index 6864d50..1ba2ada 100644
--- a/tools/src/hfst-head.cc
+++ b/tools/src/hfst-head.cc
@@ -181,6 +181,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
first_but_n.pop_front();
}
}
+ outstream.flush();
instream.close();
outstream.close();
return EXIT_SUCCESS;
diff --git a/tools/src/hfst-info.cc b/tools/src/hfst-info.cc
index 4e70ba6..4ff37e1 100644
--- a/tools/src/hfst-info.cc
+++ b/tools/src/hfst-info.cc
@@ -226,6 +226,13 @@ int main (int argc, char * argv[])
error(EXIT_FAILURE, 0, "Required foma support not present");
#endif
}
+ else if ((*f == "xfsm") || (*f == "XFSM") || (*f == "HAVE_XFSM"))
+ {
+ verbose_printf("Requiring xfsm support from library");
+#ifndef HAVE_XFSM
+ error(EXIT_FAILURE, 0, "Required xfsm support not present");
+#endif
+ }
else if ((*f == "openfst") || (*f == "OPENFST") || (*f == "HAVE_OPENFST"))
{
verbose_printf("Requiring OpenFst support from library");
@@ -266,6 +273,9 @@ int main (int argc, char * argv[])
#if HAVE_FOMA
verbose_printf("foma supported\n");
#endif
+#if HAVE_XFSM
+ verbose_printf("xfsm supported\n");
+#endif
#if USE_GLIB_UNICODE
verbose_printf("Unicode support: glib\n");
#elif USE_ICU_UNICODE
diff --git a/tools/src/hfst-lexc-compiler.cc b/tools/src/hfst-lexc-compiler.cc
index aa64185..dbd00d6 100644
--- a/tools/src/hfst-lexc-compiler.cc
+++ b/tools/src/hfst-lexc-compiler.cc
@@ -32,7 +32,12 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
#include "HfstTransducer.h"
#include "HfstOutputStream.h"
@@ -215,6 +220,7 @@ parse_options(int argc, char** argv)
}
format = hfst::TROPICAL_OPENFST_TYPE;
}
+
if (argc - optind > 0)
{
lexcfilenames = static_cast<char**>(malloc(sizeof(char*)*(argc-optind)));
@@ -248,6 +254,13 @@ lexc_streams(LexcCompiler& lexc, HfstOutputStream& outstream)
verbose_printf("Parsing lexc file %s\n", lexcfilenames[i]);
if (lexcfiles[i] == stdin)
{
+#ifdef _MSC_VER
+ if (!silent)
+ {
+ warning(0, 0, "Reading from standard input. UTF-8 characters outside\n"
+ "ascii range are supported only if input comes from a file.");
+ }
+#endif
lexc.parse(stdin);
}
else
diff --git a/tools/src/hfst-lookup.cc b/tools/src/hfst-lookup.cc
index 0f6bc32..921fe9d 100644
--- a/tools/src/hfst-lookup.cc
+++ b/tools/src/hfst-lookup.cc
@@ -22,7 +22,7 @@
#endif
#ifdef WINDOWS
-#include <io.h>
+# include <io.h>
#endif
#include <iostream>
@@ -32,7 +32,15 @@
#include <cstdlib>
#include <cstring>
#include <cstdarg>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
+
+#include "hfst-string-conversions.h"
+
#include <limits>
#include <math.h>
@@ -79,13 +87,21 @@ using hfst::StringSet;
using std::string;
using std::vector;
+using hfst::hfst_fprintf;
+
// add tools-specific variables here
static char* lookup_file_name;
static FILE* lookup_file;
-static bool pipe_mode = false;
+static bool pipe_input = false;
+static bool pipe_output = false;
static size_t linen = 0;
static bool lookup_given = false;
static size_t infinite_cutoff = 5;
+static float beam=-1;
+
+// symbols actually seen in (non-ol) transducers
+static std::vector<std::set<std::string> > cascade_symbols_seen;
+static std::vector<bool> cascade_unknown_or_identity_seen;
enum lookup_input_format
{
@@ -216,9 +232,9 @@ print_usage()
print_common_program_options(message_out);
fprintf(message_out,
"Input/Output options:\n"
- " -i, --input=INFILE Read input transducer from INFILE\n"
- " -o, --output=OUTFILE Write output to OUTFILE\n"
- " -p, --pipe-mode Read lookup strings from standard input, do not prompt\n");
+ " -i, --input=INFILE Read input transducer from INFILE\n"
+ " -o, --output=OUTFILE Write output to OUTFILE\n"
+ " -p, --pipe-mode[=STREAM] Control input and output streams\n");
fprintf(message_out, "Lookup options:\n"
" -I, --input-strings=SFILE Read lookup strings from SFILE\n"
@@ -228,6 +244,8 @@ print_usage()
" -x, --statistics Print statistics\n"
" -X, --xfst=VARIABLE Toggle xfst VARIABLE\n"
" -c, --cycles=INT How many times to follow input epsilon cycles\n"
+ " -b, --beam=B Output only analyses whose weight is within B from\n"
+ " the best analysis\n"
" -P, --progress Show neat progress bar if possible\n");
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
@@ -239,10 +257,37 @@ print_usage()
"quote-special,show-flags,obey-flags}\n"
"Input epsilon cycles are followed by default INT=5 times.\n"
"Epsilon is printed by default as an empty string.\n"
+ "B must be a non-negative float.\n"
"If the input contains several transducers, a set containing\n"
"results from all transducers is printed for each input string.\n");
fprintf(message_out, "\n");
+ fprintf(message_out, "STREAM can be { input, output, both }. If not given, defaults to {both}.\n"
+#ifdef _MSC_VER
+ "If input file is not specified with -I, input is read interactively via the\n"
+ "console, i.e. line by line from the user. If you redirect input from a file,\n"
+ "use --pipe-mode=input. Output is by default printed to the console. If you\n"
+ "redirect output to a file, use --pipe-mode=output.\n");
+#else
+ "If input file is not specified with -I, input is read interactively line by\n"
+ "line from the user. If you redirect input from a file, use --pipe-mode=input.\n"
+ "--pipe-mode=output is ignored on non-windows platforms.\n");
+#endif
+ fprintf(message_out, "\n");
+
+/* fprintf(message_out,
+#ifdef _MSC_VER
+ "If input file is not specified with -I, input is read interactively via the\n"
+ "console, i.e. line by line from the user. If input is redirected from a file,\n"
+ "use -p. Output is by default printed to the console. If output is redirected\n"
+ "to a file, use -k.\n"
+#else
+ "If input file is not specified with -I, input is read interactively line by\n"
+ "line from the user. If input is redirected from a file, use -p.\n"
+#endif
+);*/
+
+
fprintf(message_out,
"Todo:\n"
" For optimized lookup format, only strings that pass "
@@ -280,14 +325,15 @@ parse_options(int argc, char** argv)
{"xfst", required_argument, 0, 'X'},
{"epsilon-format", required_argument, 0, 'e'},
{"epsilon-format2", required_argument, 0, 'E'},
- {"pipe-mode", no_argument, 0, 'p'},
+ {"beam", required_argument, 0, 'b'},
+ {"pipe-mode", optional_argument, 0, 'p'},
{"progress", no_argument, 0, 'P'},
{0,0,0,0}
};
int option_index = 0;
// add tool-specific options here
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "I:O:F:xc:X:e:E:pP",
+ HFST_GETOPT_UNARY_SHORT "I:O:F:xc:X:e:E:b:p::P",
long_options, &option_index);
if (-1 == c)
{
@@ -351,6 +397,14 @@ parse_options(int argc, char** argv)
case 'E':
epsilon_format = hfst_strdup(optarg);
break;
+ case 'b':
+ beam = atof(optarg);
+ if (beam < 0)
+ {
+ std::cerr << "Invalid argument for --beam\n";
+ return EXIT_FAILURE;
+ }
+ break;
case 'x':
print_statistics = true;
break;
@@ -385,9 +439,22 @@ parse_options(int argc, char** argv)
case 'c':
infinite_cutoff = (size_t)atoi(hfst_strdup(optarg));
break;
+
case 'p':
- pipe_mode = true;
- break;
+ if (optarg == NULL)
+ { pipe_input = true; pipe_output = true; }
+ else if (strcmp(optarg, "both") == 0 || strcmp(optarg, "BOTH") == 0)
+ { pipe_input = true; pipe_output = true; }
+ else if (strcmp(optarg, "input") == 0 || strcmp(optarg, "INPUT") == 0 ||
+ strcmp(optarg, "in") == 0 || strcmp(optarg, "IN") == 0)
+ { pipe_input = true; }
+ else if (strcmp(optarg, "output") == 0 || strcmp(optarg, "OUTPUT") == 0 ||
+ strcmp(optarg, "out") == 0 || strcmp(optarg, "OUT") == 0)
+ { pipe_output = true; }
+ else
+ { error(EXIT_FAILURE, 0, "--pipe-mode argument %s unrecognised", optarg); }
+ break;
+
case 'P':
show_progress_bar = true;
break;
@@ -457,7 +524,7 @@ parse_options(int argc, char** argv)
static void print_prompt()
{
- if (!silent && !pipe_mode && !lookup_given)
+ if (!silent && !pipe_input && !lookup_given)
{
fprintf(stderr, "> ");
}
@@ -688,7 +755,11 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
}
else
{
+#ifdef _MSC_VER
+ w = std::numeric_limits<float>::infinity();
+#else
w = INFINITY;
+#endif
}
i = strdup(inputform);
if (lookupform != NULL)
@@ -784,8 +855,20 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
src++;
}
else if (*src == 'w')
- {
- int skip = snprintf(dst, space_left, "%f", w);
+ {
+ int skip = 0;
+#ifdef _MSC_VER
+ if (w == std::numeric_limits<float>::infinity())
+#else
+ if (false)
+#endif
+ {
+ skip = snprintf(dst, space_left, "%s", "inf");
+ }
+ else
+ {
+ skip = snprintf(dst, space_left, "%f", w);
+ }
dst += skip;
space_left -= skip;
src++;
@@ -825,10 +908,10 @@ lookup_printf(const char* format, const HfstOneLevelPath* input,
free(inputform);
free(lookupform);
int rv;
- if (not quote_special)
- rv = fprintf(ofile, "%s", res);
+ if (! quote_special)
+ rv = hfst::hfst_fprintf(ofile, "%s", res);
else
- rv = fprintf(ofile, "%s", get_print_format(res).c_str());
+ rv = hfst::hfst_fprintf(ofile, "%s", get_print_format(res).c_str());
free(res);
return rv;
}
@@ -885,7 +968,7 @@ static std::string escape_special_characters(char *s)
}
HfstOneLevelPath*
-line_to_lookup_path(char** s, HfstStrings2FstTokenizer& tok,
+line_to_lookup_path(char** s, hfst::HfstStrings2FstTokenizer& tok,
char** markup, bool* outside_sigma, bool optimized_lookup)
{
HfstOneLevelPath* rv = new HfstOneLevelPath;
@@ -936,6 +1019,7 @@ line_to_lookup_path(char** s, HfstStrings2FstTokenizer& tok,
for (StringPairVector::const_iterator it = spv.begin();
it != spv.end(); it++)
{
+ // todo: check if symbol is known to transducer
rv->second.push_back(it->first);
}
}
@@ -1083,6 +1167,24 @@ static void print_lookup_string(const StringVector &s) {
}
}
+bool is_possible_to_get_result(const HfstOneLevelPath & s,
+ const StringSet & symbols_seen,
+ bool unknown_or_identity_seen)
+{
+ if (unknown_or_identity_seen)
+ return true;
+ StringVector sv = s.second;
+ for (StringVector::const_iterator it = sv.begin(); it != sv.end(); it++)
+ {
+ if (symbols_seen.find(*it) == symbols_seen.end())
+ return false;
+ }
+ return true;
+}
+
+// which transducer in the cascade we are handling
+static unsigned int transducer_number=0;
+
void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
const HfstOneLevelPath& s, ssize_t limit = -1)
{
@@ -1092,7 +1194,11 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
HfstTwoLevelPaths results_spv;
StringPairVector path_spv;
- t.lookup_fd(s.second, results_spv, infinite_cutoff);
+ if (is_possible_to_get_result(s, cascade_symbols_seen[transducer_number],
+ cascade_unknown_or_identity_seen[transducer_number]))
+ {
+ t.lookup_fd(s.second, results_spv, infinite_cutoff);
+ }
if (print_pairs) {
@@ -1101,32 +1207,38 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
print_lookup_string(s.second);
fprintf(outfile, "\n");
}
- else {
+ else { // HERE!!!
+ float lowest_weight = -1;
/* For all result strings, */
for (HfstTwoLevelPaths::const_iterator it
= results_spv.begin(); it != results_spv.end(); it++) {
-
- /* print the lookup string */
- print_lookup_string(s.second);
- fprintf(outfile, "\t");
-
- /* and the path that yielded the result string */
- bool first_pair=true;
- for (StringPairVector::const_iterator IT = it->second.begin();
- IT != it->second.end(); IT++) {
- if (print_space && not first_pair) {
- fprintf(outfile, " ");
+
+ if (it == results_spv.begin())
+ lowest_weight = it->first;
+ if (beam < 0 || it->first <= (lowest_weight + beam))
+ {
+ /* print the lookup string */
+ print_lookup_string(s.second);
+ fprintf(outfile, "\t");
+
+ /* and the path that yielded the result string */
+ bool first_pair=true;
+ for (StringPairVector::const_iterator IT = it->second.begin();
+ IT != it->second.end(); IT++) {
+ if (print_space && ! first_pair) {
+ fprintf(outfile, " ");
+ }
+ first_pair=false;
+ fprintf(outfile, "%s:%s",
+ get_print_format(IT->first).c_str(),
+ get_print_format(IT->second).c_str());
+ }
+ /* and the weight of that path. */
+ fprintf(outfile, "\t%f\n", it->first);
}
- first_pair=false;
- fprintf(outfile, "%s:%s",
- get_print_format(IT->first).c_str(),
- get_print_format(IT->second).c_str());
- }
- /* and the weight of that path. */
- fprintf(outfile, "\t%f\n", it->first);
}
- }
fprintf(outfile, "\n");
+ }
}
// Convert HfstTwoLevelPaths into HfstOneLevelPaths
@@ -1166,12 +1278,17 @@ void lookup_fd_and_print(HfstBasicTransducer &t, HfstOneLevelPaths& results,
results = filtered;
}
+
HfstOneLevelPaths*
lookup_simple(const HfstOneLevelPath& s, HfstBasicTransducer& t, bool* infinity)
{
HfstOneLevelPaths* results = new HfstOneLevelPaths;
- if (t.is_lookup_infinitely_ambiguous(s))
+ bool possible = is_possible_to_get_result
+ (s, cascade_symbols_seen[transducer_number],
+ cascade_unknown_or_identity_seen[transducer_number]);
+
+ if (possible && t.is_lookup_infinitely_ambiguous(s))
{
if (!silent && infinite_cutoff > 0) {
warning(0, 0, "Got infinite results, number of cycles limited to " SIZE_T_SPECIFIER "",
@@ -1225,6 +1342,7 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstTransducer> cascade,
return results;
}
+
HfstOneLevelPaths*
lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> cascade,
bool* infinity)
@@ -1234,6 +1352,7 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> cascade,
// go through all transducers in the cascade
for (unsigned int i = 0; i < cascade.size(); i++)
{
+ transducer_number=i; // needed for lookup_simple
HfstOneLevelPaths* result = lookup_simple(s, cascade[i], infinity);
if (infinity)
{
@@ -1255,11 +1374,14 @@ lookup_cascading(const HfstOneLevelPath& s, vector<HfstBasicTransducer> cascade,
}
+// HERE!!! limits kvs with beam
void
print_lookups(const HfstOneLevelPaths& kvs,
const HfstOneLevelPath& kv, char* markup,
bool outside_sigma, bool inf, FILE * ofile)
{
+ float lowest_weight = -1;
+
if (outside_sigma)
{
lookup_printf(unknown_begin_setf, &kv, NULL, markup, ofile);
@@ -1283,8 +1405,13 @@ print_lookups(const HfstOneLevelPaths& kvs,
++lkv)
{
HfstOneLevelPath lup = *lkv;
- lookup_printf(infinite_lookupf, &kv, &lup, markup, ofile);
- analyses++;
+ if (lkv == kvs.begin())
+ lowest_weight = lup.first;
+ if (beam < 0 || lup.first <= (lowest_weight + beam))
+ {
+ lookup_printf(infinite_lookupf, &kv, &lup, markup, ofile);
+ analyses++;
+ }
}
lookup_printf(infinite_end_setf, &kv, NULL, markup, ofile);
}
@@ -1298,8 +1425,13 @@ print_lookups(const HfstOneLevelPaths& kvs,
++lkv)
{
HfstOneLevelPath lup = *lkv;
- lookup_printf(lookupf, &kv, &lup, markup, ofile);
- analyses++;
+ if (lkv == kvs.begin())
+ lowest_weight = lup.first;
+ if (beam < 0 || lup.first <= (lowest_weight + beam))
+ {
+ lookup_printf(lookupf, &kv, &lup, markup, ofile);
+ analyses++;
+ }
}
lookup_printf(end_setf, &kv, NULL, markup, ofile);
}
@@ -1352,7 +1484,6 @@ perform_lookups(HfstOneLevelPath& origin, std::vector<HfstBasicTransducer>& casc
return kvs;
}
-
int
process_stream(HfstInputStream& inputstream, FILE* outstream)
{
@@ -1363,11 +1494,13 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
size_t transducer_n=0;
StringVector mc_symbols;
+ bool id_or_unk_seen = false;
while (inputstream.is_good())
{
transducer_n++;
HfstTransducer trans(inputstream);
hfst::ImplementationType type = trans.get_type();
+ std::set<std::string> symbols_seen;
if (type != HFST_OL_TYPE && type != HFST_OLW_TYPE)
{
@@ -1402,6 +1535,9 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
tr_it = it->begin(); tr_it != it->end(); tr_it++)
{
std::string mcs = tr_it->get_input_symbol();
+ symbols_seen.insert(mcs);
+ if (mcs == hfst::internal_unknown || mcs == hfst::internal_identity)
+ id_or_unk_seen = true;
if (mcs.size() > 1) {
mc_symbols.push_back(mcs);
verbose_printf("multicharacter symbol: %s\n",
@@ -1410,9 +1546,15 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
}
cascade_mut.push_back(basic);
- }
+ cascade_symbols_seen.push_back(symbols_seen);
+ if (id_or_unk_seen)
+ cascade_unknown_or_identity_seen.push_back(true);
+ else
+ cascade_unknown_or_identity_seen.push_back(false);
+ }
cascade.push_back(trans);
+ id_or_unk_seen = false;
}
inputstream.close();
@@ -1430,7 +1572,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
char* line = 0;
size_t llen = 0;
- HfstStrings2FstTokenizer input_tokenizer(mc_symbols,
+ hfst::HfstStrings2FstTokenizer input_tokenizer(mc_symbols,
std::string(epsilon_format));
if (!only_optimized_lookup)
@@ -1457,10 +1599,31 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
}
print_prompt();
long filepos = ftell(lookup_file);
- while (hfst_getline(&line, &llen, lookup_file) != -1)
+ while (true)
{
+#ifdef WINDOWS
+ if (lookup_file == stdin && !pipe_input)
+ {
+ std::string str("");
+ size_t bufsize = 1000;
+ if (! hfst::get_line_from_console(str, bufsize))
+ {
+ break;
+ }
+ line = strdup(str.c_str());
+ }
+ else
+ {
+#endif
+ if (hfst_getline(&line, &llen, lookup_file) == -1)
+ break;
+#ifdef WINDOWS
+ }
+#endif
+
+ char * p = line;
linen++;
- char *p = line;
+
while (*p != '\0')
{
if (*p == '\n' || *p == '\r') // '\r' is possible on Windows
@@ -1515,7 +1678,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
unknown, &infinite);
}
- if (not print_pairs) {
+ if (! print_pairs) {
// printing was already done in function lookup_fd
print_lookups(*kvs, *kv, markup, unknown, infinite, outstream);
}
@@ -1546,6 +1709,7 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
int main( int argc, char **argv ) {
hfst_setlocale();
hfst_set_program_name(argv[0], "0.6", "HfstLookup");
+
int retval = parse_options(argc, argv);
if (retval != EXIT_CONTINUE)
{
@@ -1557,6 +1721,15 @@ int main( int argc, char **argv ) {
{
_setmode(0, _O_BINARY);
}
+
+ if (inputfile == stdin && pipe_input && show_progress_bar)
+ {
+ // todo: print warning?
+ show_progress_bar = false;
+ }
+
+ // has no effect on windows or mac
+ hfst::print_output_to_console(!pipe_output);
#endif
// close buffers, we use streams
diff --git a/tools/src/hfst-minimize.cc b/tools/src/hfst-minimize.cc
index 8c67418..1b21c77 100644
--- a/tools/src/hfst-minimize.cc
+++ b/tools/src/hfst-minimize.cc
@@ -33,6 +33,10 @@
#include <cstring>
#include <getopt.h>
+#ifdef PROFILE
+ #include <time.h>
+#endif
+
#include "hfst-commandline.h"
#include "hfst-program-options.h"
#include "hfst-tool-metadata.h"
@@ -135,12 +139,26 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
verbose_printf("Minimizing %s..." SIZE_T_SPECIFIER "\n", inputname, transducer_n);
}
+
+#ifdef PROFILE
+ bool val = hfst::get_minimize_even_if_already_minimal();
+ hfst::set_minimize_even_if_already_minimal(true);
+ clock_t t;
+ t = clock();
+#endif
trans.minimize();
+#ifdef PROFILE
+ t = clock() - t;
+ hfst::set_minimize_even_if_already_minimal(val);
+ std::cerr << "Minimization took " << ((float)t)/CLOCKS_PER_SEC << " seconds" << std::endl;
+#endif
+
hfst_set_name(trans, trans, "minimize");
hfst_set_formula(trans, trans, "M");
outstream << trans;
free(inputname);
}
+ outstream.flush();
instream.close();
outstream.close();
return EXIT_SUCCESS;
@@ -183,7 +201,13 @@ int main( int argc, char **argv ) {
try {
instream = (inputfile != stdin) ?
new HfstInputStream(inputfilename) : new HfstInputStream();
- } catch(const HfstException e) {
+ }
+ catch(const ImplementationTypeNotAvailableException & e) {
+ error(EXIT_FAILURE, 0, "file %s is in %s format which is not available",
+ inputfilename, hfst::implementation_type_to_format(instream->get_type()));
+ return EXIT_FAILURE;
+ }
+ catch(const HfstException & e) {
error(EXIT_FAILURE, 0, "%s is not a valid transducer file",
inputfilename);
return EXIT_FAILURE;
diff --git a/tools/src/hfst-optimized-lookup.cc b/tools/src/hfst-optimized-lookup.cc
index 73cc34d..7ed03fb 100644
--- a/tools/src/hfst-optimized-lookup.cc
+++ b/tools/src/hfst-optimized-lookup.cc
@@ -24,6 +24,23 @@
*/
#include "hfst-optimized-lookup.h"
+#include "inc/globals-common.h" // some cc files include extern declarations for these variables...
+
+//#ifdef _MSC_VER
+//# include <windows.h>
+//#endif
+
+#ifdef _MSC_VER
+# include "hfst-string-conversions.h"
+using hfst::hfst_fprintf;
+#endif
+
+#include <cstdarg>
+#include <iostream> // DEBUG
+
+static float beam=-1;
+static bool pipe_input = false;
+static bool pipe_output = false;
bool print_usage(void)
{
@@ -43,15 +60,27 @@ bool print_usage(void)
" -u, --unique Suppress duplicate analyses\n" <<
" -n N, --analyses=N Output no more than N analyses\n" <<
" (if the transducer is weighted, the N best analyses)\n" <<
+ " -b, --beam=B Output only analyses whose weight is within B from\n" <<
+ " the best analysis\n" <<
" -x, --xerox Xerox output format (default)\n" <<
" -f, --fast Be as fast as possible.\n" <<
" (with this option enabled -u and -n don't work and\n" <<
" output won't be ordered by weight).\n" <<
+ " -p, --pipe-mode[=STREAM] Control input and output streams.\n" <<
"\n" <<
- "Note that " << PACKAGE_NAME << " is *not* guaranteed to behave identically to\n" <<
- "hfst-lookup (although it almost always does): input-side multicharacter symbols\n" <<
- "are not fully supported. If the first character of such a symbol is an ASCII\n" <<
- "symbol also matching a single-character symbol, it will be tokenized as such.\n" <<
+ "N must be a positive integer. B must be a non-negative float.\n" <<
+ "Options -n and -b are combined with AND, i.e. they both restrict the output.\n" <<
+ "\n" <<
+ "STREAM can be { input, output, both }. If not given, defaults to {both}.\n" <<
+#ifdef _MSC_VER
+ "Input is read interactively via the console, i.e. line by line from the user.\n" <<
+ "If you redirect input from a file, use --pipe-mode=input. Output is by default\n" <<
+ "printed to the console. If you redirect output to a file, use --pipe-mode=output.\n" <<
+#else
+ "Input is read interactively line by line from the user. If you redirect input\n" <<
+ "from a file, use --pipe-mode=input. --pipe-mode=output is ignored on non-windows\n" <<
+ "platforms.\n" <<
+#endif
"\n" <<
"Report bugs to " << PACKAGE_BUGREPORT << "\n" <<
"\n";
@@ -76,8 +105,9 @@ bool print_short_help(void)
int main(int argc, char **argv)
{
+
int c;
-
+
while (true)
{
static struct option long_options[] =
@@ -91,15 +121,17 @@ int main(int argc, char **argv)
// the hfst-optimized-lookup-specific options
{"echo-inputs", no_argument, 0, 'e'},
{"show-weights", no_argument, 0, 'w'},
+ {"beam", required_argument, 0, 'b'},
{"unique", no_argument, 0, 'u'},
{"xerox", no_argument, 0, 'x'},
{"fast", no_argument, 0, 'f'},
+ {"pipe-mode", optional_argument, 0, 'p'},
{"analyses", required_argument, 0, 'n'},
{0, 0, 0, 0 }
};
int option_index = 0;
- c = getopt_long(argc, argv, "hVvqsewuxfn:", long_options, &option_index);
+ c = getopt_long(argc, argv, "hVvqsewb:uxfn:p::", long_options, &option_index);
if (c == -1) // no more options to look at
break;
@@ -153,6 +185,15 @@ int main(int argc, char **argv)
displayUniqueFlag = true;
break;
+ case 'b':
+ beam = atof(optarg);
+ if (beam < 0)
+ {
+ std::cerr << "Invalid argument for --beam\n";
+ return EXIT_FAILURE;
+ }
+ break;
+
case 'n':
maxAnalyses = atoi(optarg);
if (maxAnalyses < 1)
@@ -169,6 +210,22 @@ int main(int argc, char **argv)
case 'f':
beFast = true;
break;
+
+ case 'p':
+ if (optarg == NULL)
+ { pipe_input = true; pipe_output = true; }
+ else if (strcmp(optarg, "both") == 0 || strcmp(optarg, "BOTH") == 0)
+ { pipe_input = true; pipe_output = true; }
+ else if (strcmp(optarg, "input") == 0 || strcmp(optarg, "INPUT") == 0 ||
+ strcmp(optarg, "in") == 0 || strcmp(optarg, "IN") == 0)
+ { pipe_input = true; }
+ else if (strcmp(optarg, "output") == 0 || strcmp(optarg, "OUTPUT") == 0 ||
+ strcmp(optarg, "out") == 0 || strcmp(optarg, "OUT") == 0)
+ { pipe_output = true; }
+ else
+ { std::cerr << "--pipe-mode argument " << std::string(optarg) << " unrecognised\n\n";
+ return EXIT_FAILURE; }
+ break;
default:
std::cerr << "Invalid option\n\n";
@@ -177,6 +234,11 @@ int main(int argc, char **argv)
break;
}
}
+
+#ifdef _MSC_VER
+ hfst::print_output_to_console(!pipe_output); // has no effect on windows or mac
+#endif
+
// no more options, we should now be at the input filename
if ( (optind + 1) < argc)
{
@@ -325,6 +387,11 @@ void LetterTrie::add_string(const char * p, SymbolNumber symbol_key)
letters[(unsigned char)(*p)]->add_string(p+1,symbol_key);
}
+bool LetterTrie::has_key_starting_with(const char c) const
+{
+ return letters[(unsigned char) c] != NULL;
+}
+
SymbolNumber LetterTrie::find_key(char ** p)
{
const char * old_p = *p;
@@ -350,10 +417,18 @@ void Encoder::read_input_symbols(KeyTable * kt)
assert(kt->find(k) != kt->end());
#endif
const char * p = kt->operator[](k);
- if ((strlen(p) == 1) && (unsigned char)(*p) <= 127)
- {
+ if ((strlen(p) == 1) && (unsigned char)(*p) <= 127
+ // we have a single char ascii symbol
+ && !letters.has_key_starting_with(*p)) {
+ // make sure there isn't a longer symbol we would be shadowing
ascii_symbols[(unsigned char)(*p)] = k;
- }
+ }
+ // If there's an ascii tokenized symbol shadowing this, remove it
+ if (strlen(p) > 1 &&
+ (unsigned char)(*p) <= 127 &&
+ ascii_symbols[(unsigned char)(*p)] != NO_SYMBOL_NUMBER) {
+ ascii_symbols[(unsigned char)(*p)] = NO_SYMBOL_NUMBER;
+ }
letters.add_string(p,k);
}
}
@@ -377,21 +452,37 @@ void runTransducer (genericTransducer T)
{
input_string[i] = NO_SYMBOL_NUMBER;
}
-
+
char * str = (char*)(malloc(MAX_IO_STRING*sizeof(char)));
*str = 0;
char * old_str = str;
-
- while(std::cin.getline(str,MAX_IO_STRING))
+
+ while(true)
{
- // Carriage returns in Windows..
- unsigned int last_char_index = (unsigned int) (std::cin.gcount() > 2)? (std::cin.gcount()) - 2: 0;
- if (str[last_char_index] == '\r')
- str[last_char_index] = '\0';
-
+#ifdef WINDOWS
+ if (!pipe_input)
+ {
+ free(str);
+ std::string linestr("");
+ if (! hfst::get_line_from_console(linestr, MAX_IO_STRING*sizeof(char)))
+ break;
+ str = strdup(linestr.c_str());
+ old_str = str;
+ }
+ else
+#endif
+ {
+ if (! std::cin.getline(str,MAX_IO_STRING))
+ break;
+ }
+
if (echoInputsFlag)
{
+#ifndef WINDOWS
std::cout << str << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\n", str); // fix: add \r?
+#endif
}
int i = 0;
SymbolNumber k = NO_SYMBOL_NUMBER;
@@ -420,7 +511,11 @@ void runTransducer (genericTransducer T)
{ // tokenization failed
if (outputType == xerox)
{
+#ifndef WINDOWS
std::cout << str << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?\n", str);
+#endif
std::cout << std::endl;
}
continue;
@@ -435,6 +530,7 @@ void runTransducer (genericTransducer T)
int setup(FILE * f)
{
+ try {
TransducerHeader header(f);
TransducerAlphabet alphabet(f, header.symbol_count());
@@ -499,6 +595,14 @@ int setup(FILE * f)
}
}
}
+ }
+ catch (const HeaderParsingException & e)
+ {
+ std::cerr << "Invalid transducer header.\n";
+ std::cerr << "The transducer must be in optimized lookup format.\n";
+ return EXIT_FAILURE;
+ }
+
return 0;
}
@@ -821,7 +925,11 @@ void Transducer::note_analysis(SymbolNumber * whole_output_string)
{
for (SymbolNumber * num = whole_output_string; *num != NO_SYMBOL_NUMBER; ++num)
{
+#ifndef WINDOWS
std::cout << symbol_table[*num];
+#else
+ hfst_fprintf(stdout, "%s", symbol_table[*num]);
+#endif
}
std::cout << std::endl;
} else
@@ -936,7 +1044,11 @@ void Transducer::printAnalyses(std::string prepend)
{
if (outputType == xerox && display_vector.size() == 0)
{
+#ifndef WINDOWS
std::cout << prepend << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?\n", prepend.c_str());
+#endif
std::cout << std::endl;
return;
}
@@ -946,9 +1058,17 @@ void Transducer::printAnalyses(std::string prepend)
{
if (outputType == xerox)
{
+#ifndef WINDOWS
std::cout << prepend << "\t";
- }
+#else
+ hfst_fprintf(stdout, "%s\t", prepend.c_str());
+#endif
+ }
+#ifndef WINDOWS
std::cout << *it << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\n", it->c_str());
+#endif
++it;
++i;
}
@@ -961,7 +1081,11 @@ void TransducerUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox && display_vector.size() == 0)
{
+#ifndef WINDOWS
std::cout << prepend << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?\n", prepend.c_str());
+#endif
std::cout << std::endl;
return;
}
@@ -971,9 +1095,17 @@ void TransducerUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox)
{
+#ifndef WINDOWS
std::cout << prepend << "\t";
+#else
+ hfst_fprintf(stdout, "%s\t", prepend.c_str());
+#endif
}
+#ifndef WINDOWS
std::cout << *it << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\n", it->c_str());
+#endif
++it;
++i;
}
@@ -985,7 +1117,11 @@ void TransducerFdUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox && display_vector.size() == 0)
{
+#ifndef WINDOWS
std::cout << prepend << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?\n", prepend.c_str());
+#endif
std::cout << std::endl;
return;
}
@@ -995,9 +1131,17 @@ void TransducerFdUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox)
{
+#ifndef WINDOWS
std::cout << prepend << "\t";
+#else
+ hfst_fprintf(stdout, "%s\t", prepend.c_str());
+#endif
}
+#ifndef WINDOWS
std::cout << *it << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\n", it->c_str());
+#endif
++it;
++i;
}
@@ -1198,7 +1342,7 @@ void TransducerW::try_epsilon_transitions(SymbolNumber * input_symbol,
return;
}
- while ((transitions[i] != NULL) and (transitions[i]->get_input() == 0))
+ while ((transitions[i] != NULL) && (transitions[i]->get_input() == 0))
{
*output_symbol = transitions[i]->get_output();
current_weight += transitions[i]->get_weight();
@@ -1393,24 +1537,44 @@ void TransducerW::printAnalyses(std::string prepend)
{
if (outputType == xerox && display_map.size() == 0)
{
+#ifndef WINDOWS
std::cout << prepend << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?\n", prepend.c_str());
+#endif
std::cout << std::endl;
return;
}
int i = 0;
+ float lowest_weight = -1;
DisplayMultiMap::iterator it = display_map.begin();
while ( (it != display_map.end()) && (i < maxAnalyses))
{
- if (outputType == xerox)
- {
- std::cout << prepend << "\t";
- }
- std::cout << (*it).second;
- if (displayWeightsFlag)
- {
- std::cout << '\t' << (*it).first;
- }
- std::cout << std::endl;
+ if (it == display_map.begin())
+ lowest_weight = it->first;
+ // if beam is not set, i.e. has a negative value (-1.0), the only constraint
+ // is maxAnalyses
+ if (beam < 0 || it->first <= (lowest_weight + beam))
+ {
+ if (outputType == xerox)
+ {
+#ifndef WINDOWS
+ std::cout << prepend << "\t";
+#else
+ hfst_fprintf(stdout, "%s\t", prepend.c_str());
+#endif
+ }
+#ifndef WINDOWS
+ std::cout << (*it).second;
+#else
+ hfst_fprintf(stdout, "%s", (*it).second.c_str());
+#endif
+ if (displayWeightsFlag)
+ {
+ std::cout << '\t' << (*it).first;
+ }
+ std::cout << std::endl;
+ }
++it;
++i;
}
@@ -1422,16 +1586,24 @@ void TransducerWUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox && display_map.size() == 0)
{
+#ifndef WINDOWS
std::cout << prepend << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?\n", prepend.c_str());
+#endif
std::cout << std::endl;
return;
}
int i = 0;
+ float lowest_weight = -1 ;
std::multimap<Weight, std::string> weight_sorted_map;
DisplayMap::iterator it = display_map.begin();
while (it != display_map.end())
{
- weight_sorted_map.insert(std::pair<Weight, std::string>((*it).second, (*it).first));
+ if (it == display_map.begin())
+ lowest_weight = it->second;
+ if (beam < 0 || it->second <= (lowest_weight + beam))
+ weight_sorted_map.insert(std::pair<Weight, std::string>((*it).second, (*it).first));
++it;
}
std::multimap<Weight, std::string>::iterator display_it = weight_sorted_map.begin();
@@ -1439,9 +1611,17 @@ void TransducerWUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox)
{
+#ifndef WINDOWS
std::cout << prepend << "\t";
+#else
+ hfst_fprintf(stdout, "%s\t", prepend.c_str());
+#endif
}
+#ifndef WINDOWS
std::cout << (*display_it).second;
+#else
+ hfst_fprintf(stdout, "%s", (*display_it).second.c_str());
+#endif
if (displayWeightsFlag)
{
std::cout << '\t' << (*display_it).first;
@@ -1458,16 +1638,24 @@ void TransducerWFdUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox && display_map.size() == 0)
{
+#ifndef WINDOWS
std::cout << prepend << "\t+?" << std::endl;
+#else
+ hfst_fprintf(stdout, "%s\t+?", prepend);
+#endif
std::cout << std::endl;
return;
}
int i = 0;
+ float lowest_weight = -1 ;
std::multimap<Weight, std::string> weight_sorted_map;
DisplayMap::iterator it;
for (it = display_map.begin(); it != display_map.end(); it++)
{
- weight_sorted_map.insert(std::pair<Weight, std::string>((*it).second, (*it).first));
+ if (it == display_map.begin())
+ lowest_weight = it->second;
+ if (beam < 0 || it->second <= (lowest_weight + beam))
+ weight_sorted_map.insert(std::pair<Weight, std::string>((*it).second, (*it).first));
}
std::multimap<Weight, std::string>::iterator display_it;
for (display_it = weight_sorted_map.begin();
@@ -1476,7 +1664,11 @@ void TransducerWFdUniq::printAnalyses(std::string prepend)
{
if (outputType == xerox)
{
+#ifndef WINDOWS
std::cout << prepend << "\t";
+#else
+ hfst_fprintf(stdout, "%s\t", prepend);
+#endif
}
std::cout << (*display_it).second;
if (displayWeightsFlag)
diff --git a/tools/src/hfst-optimized-lookup.h b/tools/src/hfst-optimized-lookup.h
index 47b6797..2c4c62f 100644
--- a/tools/src/hfst-optimized-lookup.h
+++ b/tools/src/hfst-optimized-lookup.h
@@ -1,4 +1,4 @@
- /*
+/*
Copyright 2009 University of Helsinki
@@ -17,8 +17,8 @@
*/
/*
-config.h-defined constants
- */
+ config.h-defined constants
+*/
const char * PACKAGE_NAME = "hfst-optimized-lookup";
const char * PACKAGE_BUGREPORT = "hfst-bugs at helsinki.fi";
@@ -28,9 +28,14 @@ const char * PACKAGE_STRING = "hfst-optimized-lookup 1.2";
NOTE:
THIS SINGLE-FILE VERSION WAS PUT TOGETHER FROM A MULTI-FILE VERSION
SO THE CURRENT STRUCTURE IS NOT SO GREAT. TODO: FIX THIS.
- */
+*/
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
-#include <getopt.h>
#include <cstdio>
#include <vector>
#include <map>
@@ -41,6 +46,7 @@ const char * PACKAGE_STRING = "hfst-optimized-lookup 1.2";
#include <cassert>
#include <ctime>
#include <iostream>
+#include <string>
enum OutputType {HFST, xerox};
@@ -82,9 +88,9 @@ class Transition;
enum FlagDiacriticOperator {P, N, R, D, C, U};
enum HeaderFlag {Weighted, Deterministic, Input_deterministic, Minimized,
- Cyclic, Has_epsilon_epsilon_transitions,
- Has_input_epsilon_transitions, Has_input_epsilon_cycles,
- Has_unweighted_input_epsilon_cycles};
+ Cyclic, Has_epsilon_epsilon_transitions,
+ Has_input_epsilon_transitions, Has_input_epsilon_cycles,
+ Has_unweighted_input_epsilon_cycles};
typedef std::vector<TransitionIndex*> TransitionIndexVector;
typedef std::vector<Transition*> TransitionVector;
@@ -97,137 +103,137 @@ class HeaderParsingException: public std::exception
{
public:
virtual const char* what() const throw()
- { return("Parsing error while reading header"); }
+ { return("Parsing error while reading header"); }
};
class TransducerHeader
{
- private:
- SymbolNumber number_of_symbols;
- SymbolNumber number_of_input_symbols;
- TransitionTableIndex size_of_transition_index_table;
- TransitionTableIndex size_of_transition_target_table;
-
- StateIdNumber number_of_states;
- TransitionNumber number_of_transitions;
-
- bool weighted;
- bool deterministic;
- bool input_deterministic;
- bool minimized;
- bool cyclic;
- bool has_epsilon_epsilon_transitions;
- bool has_input_epsilon_transitions;
- bool has_input_epsilon_cycles;
- bool has_unweighted_input_epsilon_cycles;
-
- void read_property(bool &property, FILE * f)
- {
- unsigned int prop;
- unsigned int ret;
- ret = fread(&prop,sizeof(unsigned int),1,f);
- (void)ret;
- if (prop == 0)
- {
- property = false;
- return;
- }
- else
- {
- property = true;
- return;
- }
- std::cerr << "Could not parse transducer; wrong or corrupt file?" << std::endl;
- exit(1);
- }
-
- public:
- TransducerHeader(FILE * f)
- {
- skip_hfst3_header(f);
-
- // The silly compiler complains about not catching the return value
- // of fread(). Hence this dummy variable is needed.
- size_t val;
-
- val = fread(&number_of_input_symbols,sizeof(SymbolNumber),1,f);
- val = fread(&number_of_symbols,sizeof(SymbolNumber),1,f);
-
- val = fread(&size_of_transition_index_table,sizeof(TransitionTableIndex),1,f);
- val = fread(&size_of_transition_target_table,sizeof(TransitionTableIndex),1,f);
-
- val = fread(&number_of_states,sizeof(StateIdNumber),1,f);
- val = fread(&number_of_transitions,sizeof(TransitionNumber),1,f);
-
- (void)val;
-
- read_property(weighted,f);
-
- read_property(deterministic,f);
- read_property(input_deterministic,f);
- read_property(minimized,f);
- read_property(cyclic,f);
- read_property(has_epsilon_epsilon_transitions,f);
- read_property(has_input_epsilon_transitions,f);
- read_property(has_input_epsilon_cycles,f);
- read_property(has_unweighted_input_epsilon_cycles,f);
- }
+private:
+ SymbolNumber number_of_symbols;
+ SymbolNumber number_of_input_symbols;
+ TransitionTableIndex size_of_transition_index_table;
+ TransitionTableIndex size_of_transition_target_table;
+
+ StateIdNumber number_of_states;
+ TransitionNumber number_of_transitions;
+
+ bool weighted;
+ bool deterministic;
+ bool input_deterministic;
+ bool minimized;
+ bool cyclic;
+ bool has_epsilon_epsilon_transitions;
+ bool has_input_epsilon_transitions;
+ bool has_input_epsilon_cycles;
+ bool has_unweighted_input_epsilon_cycles;
+
+ void read_property(bool &property, FILE * f)
+ {
+ unsigned int prop;
+ unsigned int ret;
+ ret = fread(&prop,sizeof(unsigned int),1,f);
+ (void)ret;
+ if (prop == 0)
+ {
+ property = false;
+ return;
+ }
+ else
+ {
+ property = true;
+ return;
+ }
+ std::cerr << "Could not parse transducer; wrong or corrupt file?" << std::endl;
+ exit(1);
+ }
- void skip_hfst3_header(FILE * f);
-
- SymbolNumber symbol_count(void)
- { return number_of_symbols; }
-
- SymbolNumber input_symbol_count(void)
- { return number_of_input_symbols; }
- TransitionTableIndex index_table_size(void)
- { return size_of_transition_index_table; }
-
- TransitionTableIndex target_table_size(void)
- { return size_of_transition_target_table; }
-
- bool probe_flag(HeaderFlag flag)
- {
- switch (flag) {
- case Weighted: return weighted;
- case Deterministic: return deterministic;
- case Input_deterministic: return input_deterministic;
- case Minimized: return minimized;
- case Cyclic: return cyclic;
- case Has_epsilon_epsilon_transitions: return has_epsilon_epsilon_transitions;
- case Has_input_epsilon_transitions: return has_input_epsilon_transitions;
- case Has_input_epsilon_cycles: return has_input_epsilon_cycles;
- case Has_unweighted_input_epsilon_cycles: return has_unweighted_input_epsilon_cycles;
- }
- return false;
- }
+public:
+ TransducerHeader(FILE * f)
+ {
+ skip_hfst3_header(f);
+
+ // The silly compiler complains about not catching the return value
+ // of fread(). Hence this dummy variable is needed.
+ size_t val;
+
+ val = fread(&number_of_input_symbols,sizeof(SymbolNumber),1,f);
+ val = fread(&number_of_symbols,sizeof(SymbolNumber),1,f);
+
+ val = fread(&size_of_transition_index_table,sizeof(TransitionTableIndex),1,f);
+ val = fread(&size_of_transition_target_table,sizeof(TransitionTableIndex),1,f);
+
+ val = fread(&number_of_states,sizeof(StateIdNumber),1,f);
+ val = fread(&number_of_transitions,sizeof(TransitionNumber),1,f);
+
+ (void)val;
+
+ read_property(weighted,f);
+
+ read_property(deterministic,f);
+ read_property(input_deterministic,f);
+ read_property(minimized,f);
+ read_property(cyclic,f);
+ read_property(has_epsilon_epsilon_transitions,f);
+ read_property(has_input_epsilon_transitions,f);
+ read_property(has_input_epsilon_cycles,f);
+ read_property(has_unweighted_input_epsilon_cycles,f);
+ }
+
+ void skip_hfst3_header(FILE * f);
+
+ SymbolNumber symbol_count(void)
+ { return number_of_symbols; }
+
+ SymbolNumber input_symbol_count(void)
+ { return number_of_input_symbols; }
+ TransitionTableIndex index_table_size(void)
+ { return size_of_transition_index_table; }
+
+ TransitionTableIndex target_table_size(void)
+ { return size_of_transition_target_table; }
+
+ bool probe_flag(HeaderFlag flag)
+ {
+ switch (flag) {
+ case Weighted: return weighted;
+ case Deterministic: return deterministic;
+ case Input_deterministic: return input_deterministic;
+ case Minimized: return minimized;
+ case Cyclic: return cyclic;
+ case Has_epsilon_epsilon_transitions: return has_epsilon_epsilon_transitions;
+ case Has_input_epsilon_transitions: return has_input_epsilon_transitions;
+ case Has_input_epsilon_cycles: return has_input_epsilon_cycles;
+ case Has_unweighted_input_epsilon_cycles: return has_unweighted_input_epsilon_cycles;
+ }
+ return false;
+ }
};
class FlagDiacriticOperation
{
- private:
- FlagDiacriticOperator operation;
- SymbolNumber feature;
- ValueNumber value;
- public:
- FlagDiacriticOperation(FlagDiacriticOperator op, SymbolNumber feat, ValueNumber val):
- operation(op), feature(feat), value(val) {}
-
- // dummy constructor
- FlagDiacriticOperation():
- operation(P), feature(NO_SYMBOL_NUMBER), value(0) {}
+private:
+ FlagDiacriticOperator operation;
+ SymbolNumber feature;
+ ValueNumber value;
+public:
+ FlagDiacriticOperation(FlagDiacriticOperator op, SymbolNumber feat, ValueNumber val):
+ operation(op), feature(feat), value(val) {}
+
+ // dummy constructor
+ FlagDiacriticOperation():
+ operation(P), feature(NO_SYMBOL_NUMBER), value(0) {}
- bool isFlag(void) { return feature != NO_SYMBOL_NUMBER; }
- FlagDiacriticOperator Operation(void) { return operation; }
- SymbolNumber Feature(void) { return feature; }
- ValueNumber Value(void) { return value; }
+ bool isFlag(void) { return feature != NO_SYMBOL_NUMBER; }
+ FlagDiacriticOperator Operation(void) { return operation; }
+ SymbolNumber Feature(void) { return feature; }
+ ValueNumber Value(void) { return value; }
#if OL_FULL_DEBUG
- void print(void)
- {
- std::cout << operation << "\t" << feature << "\t" << value << std::endl;
- }
+ void print(void)
+ {
+ std::cout << operation << "\t" << feature << "\t" << value << std::endl;
+ }
#endif
};
@@ -235,47 +241,47 @@ typedef std::vector<FlagDiacriticOperation> OperationVector;
class TransducerAlphabet
{
- private:
- SymbolNumber number_of_symbols;
- KeyTable * kt;
- OperationVector operations;
+private:
+ SymbolNumber number_of_symbols;
+ KeyTable * kt;
+ OperationVector operations;
- void get_next_symbol(FILE * f, SymbolNumber k);
+ void get_next_symbol(FILE * f, SymbolNumber k);
- char * line;
+ char * line;
- std::map<std::string, SymbolNumber> feature_bucket;
- std::map<std::string, ValueNumber> value_bucket;
- ValueNumber val_num;
- SymbolNumber feat_num;
+ std::map<std::string, SymbolNumber> feature_bucket;
+ std::map<std::string, ValueNumber> value_bucket;
+ ValueNumber val_num;
+ SymbolNumber feat_num;
- public:
- TransducerAlphabet(FILE * f,SymbolNumber symbol_number):
- number_of_symbols(symbol_number),
- kt(new KeyTable),
- operations(),
- line((char*)(malloc(1000)))
- {
- feat_num = 0;
- val_num = 1;
- value_bucket[std::string()] = 0; // empty value = neutral
- for (SymbolNumber k = 0; k < number_of_symbols; ++k)
- {
- get_next_symbol(f,k);
- }
- // assume the first symbol is epsilon which we don't want to print
- kt->operator[](0) = "";
- free(line);
- }
+public:
+ TransducerAlphabet(FILE * f,SymbolNumber symbol_number):
+ number_of_symbols(symbol_number),
+ kt(new KeyTable),
+ operations(),
+ line((char*)(malloc(1000)))
+ {
+ feat_num = 0;
+ val_num = 1;
+ value_bucket[std::string()] = 0; // empty value = neutral
+ for (SymbolNumber k = 0; k < number_of_symbols; ++k)
+ {
+ get_next_symbol(f,k);
+ }
+ // assume the first symbol is epsilon which we don't want to print
+ kt->operator[](0) = "";
+ free(line);
+ }
- KeyTable * get_key_table(void)
- { return kt; }
+ KeyTable * get_key_table(void)
+ { return kt; }
- OperationVector get_operation_vector(void)
- { return operations; }
+ OperationVector get_operation_vector(void)
+ { return operations; }
- SymbolNumber get_state_size(void)
- { return feature_bucket.size(); }
+ SymbolNumber get_state_size(void)
+ { return feature_bucket.size(); }
};
@@ -284,40 +290,41 @@ typedef std::vector<LetterTrie*> LetterTrieVector;
class LetterTrie
{
- private:
- LetterTrieVector letters;
- SymbolNumberVector symbols;
+private:
+ LetterTrieVector letters;
+ SymbolNumberVector symbols;
- public:
- LetterTrie(void):
- letters(UCHAR_MAX, (LetterTrie*) NULL),
- symbols(UCHAR_MAX,NO_SYMBOL_NUMBER)
- {}
+public:
+ LetterTrie(void):
+ letters(UCHAR_MAX, (LetterTrie*) NULL),
+ symbols(UCHAR_MAX,NO_SYMBOL_NUMBER)
+ {}
- void add_string(const char * p,SymbolNumber symbol_key);
+ void add_string(const char * p,SymbolNumber symbol_key);
+ bool has_key_starting_with(const char c) const;
- SymbolNumber find_key(char ** p);
+ SymbolNumber find_key(char ** p);
};
class Encoder {
- private:
- SymbolNumber number_of_input_symbols;
- LetterTrie letters;
- SymbolNumberVector ascii_symbols;
+private:
+ SymbolNumber number_of_input_symbols;
+ LetterTrie letters;
+ SymbolNumberVector ascii_symbols;
- void read_input_symbols(KeyTable * kt);
+ void read_input_symbols(KeyTable * kt);
- public:
- Encoder(KeyTable * kt, SymbolNumber input_symbol_count):
- number_of_input_symbols(input_symbol_count),
- ascii_symbols(UCHAR_MAX,NO_SYMBOL_NUMBER)
- {
- read_input_symbols(kt);
- }
+public:
+ Encoder(KeyTable * kt, SymbolNumber input_symbol_count):
+ number_of_input_symbols(input_symbol_count),
+ ascii_symbols(UCHAR_MAX,NO_SYMBOL_NUMBER)
+ {
+ read_input_symbols(kt);
+ }
- SymbolNumber find_key(char ** p);
+ SymbolNumber find_key(char ** p);
};
typedef std::vector<ValueNumber> FlagDiacriticState;
@@ -335,338 +342,338 @@ typedef std::set<std::string> DisplaySet;
class TransitionIndex
{
- protected:
- SymbolNumber input_symbol;
- TransitionTableIndex first_transition_index;
+protected:
+ SymbolNumber input_symbol;
+ TransitionTableIndex first_transition_index;
- public:
+public:
- // Each TransitionIndex has an input symbol and a target index.
- static const size_t SIZE =
- sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
+ // Each TransitionIndex has an input symbol and a target index.
+ static const size_t SIZE =
+ sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
- TransitionIndex(SymbolNumber input,
- TransitionTableIndex first_transition):
- input_symbol(input),
- first_transition_index(first_transition)
- {}
+ TransitionIndex(SymbolNumber input,
+ TransitionTableIndex first_transition):
+ input_symbol(input),
+ first_transition_index(first_transition)
+ {}
- bool matches(SymbolNumber s);
+ bool matches(SymbolNumber s);
- TransitionTableIndex target(void)
- {
- return first_transition_index;
- }
+ TransitionTableIndex target(void)
+ {
+ return first_transition_index;
+ }
- bool final(void)
- {
- return first_transition_index == 1;
- }
+ bool final(void)
+ {
+ return first_transition_index == 1;
+ }
- SymbolNumber get_input(void)
- {
- return input_symbol;
- }
+ SymbolNumber get_input(void)
+ {
+ return input_symbol;
+ }
};
class Transition
{
- protected:
- SymbolNumber input_symbol;
- SymbolNumber output_symbol;
-
- TransitionTableIndex target_index;
-
- public:
-
- // Each transition has an input symbol an output symbol and
- // a target index.
- static const size_t SIZE =
- 2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
-
- Transition(SymbolNumber input,
- SymbolNumber output,
- TransitionTableIndex target):
- input_symbol(input),
- output_symbol(output),
- target_index(target)
- {}
-
- bool matches(SymbolNumber s);
-
- TransitionTableIndex target(void)
- {
- return target_index;
- }
-
- SymbolNumber get_output(void)
- {
- return output_symbol;
- }
-
- SymbolNumber get_input(void)
- {
- return input_symbol;
- }
+protected:
+ SymbolNumber input_symbol;
+ SymbolNumber output_symbol;
+
+ TransitionTableIndex target_index;
+
+public:
+
+ // Each transition has an input symbol an output symbol and
+ // a target index.
+ static const size_t SIZE =
+ 2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
+
+ Transition(SymbolNumber input,
+ SymbolNumber output,
+ TransitionTableIndex target):
+ input_symbol(input),
+ output_symbol(output),
+ target_index(target)
+ {}
+
+ bool matches(SymbolNumber s);
+
+ TransitionTableIndex target(void)
+ {
+ return target_index;
+ }
+
+ SymbolNumber get_output(void)
+ {
+ return output_symbol;
+ }
+
+ SymbolNumber get_input(void)
+ {
+ return input_symbol;
+ }
- bool final(void)
- {
- return target_index == 1;
- }
+ bool final(void)
+ {
+ return target_index == 1;
+ }
};
class IndexTableReader
{
- private:
- TransitionTableIndex number_of_table_entries;
- char * TableIndices;
- TransitionIndexVector indices;
- size_t table_size;
+private:
+ TransitionTableIndex number_of_table_entries;
+ char * TableIndices;
+ TransitionIndexVector indices;
+ size_t table_size;
- void get_index_vector(void);
- public:
- IndexTableReader(FILE * f,
- TransitionTableIndex index_count):
- number_of_table_entries(index_count)
- {
- table_size = number_of_table_entries*TransitionIndex::SIZE;
- TableIndices = (char*)(malloc(table_size));
-
- // This dummy variable is needed, since the compiler complains
- // for not catching the return value of fread().
- int dummy_number_of_bytes;
-
- dummy_number_of_bytes = fread(TableIndices,table_size,1,f);
- (void)dummy_number_of_bytes;
- get_index_vector();
- }
+ void get_index_vector(void);
+public:
+ IndexTableReader(FILE * f,
+ TransitionTableIndex index_count):
+ number_of_table_entries(index_count)
+ {
+ table_size = number_of_table_entries*TransitionIndex::SIZE;
+ TableIndices = (char*)(malloc(table_size));
+
+ // This dummy variable is needed, since the compiler complains
+ // for not catching the return value of fread().
+ int dummy_number_of_bytes;
+
+ dummy_number_of_bytes = fread(TableIndices,table_size,1,f);
+ (void)dummy_number_of_bytes;
+ get_index_vector();
+ }
- bool get_finality(TransitionTableIndex i)
- {
- return indices[i]->final();
- }
+ bool get_finality(TransitionTableIndex i)
+ {
+ return indices[i]->final();
+ }
- TransitionIndex * at(TransitionTableIndex i)
- {
- return indices[i];
- }
+ TransitionIndex * at(TransitionTableIndex i)
+ {
+ return indices[i];
+ }
- TransitionIndexVector &operator() (void)
- { return indices; }
+ TransitionIndexVector &operator() (void)
+ { return indices; }
};
class TransitionTableReader
{
- protected:
- TransitionTableIndex number_of_table_entries;
- char * TableTransitions;
- TransitionVector transitions;
- size_t table_size;
- size_t transition_size;
+protected:
+ TransitionTableIndex number_of_table_entries;
+ char * TableTransitions;
+ TransitionVector transitions;
+ size_t table_size;
+ size_t transition_size;
- TransitionTableIndex position;
+ TransitionTableIndex position;
- void get_transition_vector(void);
- public:
- TransitionTableReader(FILE * f,
- TransitionTableIndex transition_count):
- number_of_table_entries(transition_count),
- position(0)
- {
- table_size = number_of_table_entries*Transition::SIZE;
- TableTransitions = (char*)(malloc(table_size));
- int bytes;
- bytes = fread(TableTransitions,table_size,1,f);
- (void)bytes;
- get_transition_vector();
-
- }
+ void get_transition_vector(void);
+public:
+ TransitionTableReader(FILE * f,
+ TransitionTableIndex transition_count):
+ number_of_table_entries(transition_count),
+ position(0)
+ {
+ table_size = number_of_table_entries*Transition::SIZE;
+ TableTransitions = (char*)(malloc(table_size));
+ int bytes;
+ bytes = fread(TableTransitions,table_size,1,f);
+ (void)bytes;
+ get_transition_vector();
+
+ }
- void Set(TransitionTableIndex pos);
+ void Set(TransitionTableIndex pos);
- Transition * at(TransitionTableIndex i)
- {
- return transitions[i - TRANSITION_TARGET_TABLE_START];
- }
+ Transition * at(TransitionTableIndex i)
+ {
+ return transitions[i - TRANSITION_TARGET_TABLE_START];
+ }
- void Next(void)
- {
- ++position;
- }
+ void Next(void)
+ {
+ ++position;
+ }
- bool Matches(SymbolNumber s);
+ bool Matches(SymbolNumber s);
- TransitionTableIndex get_target(void)
- {
- return transitions[position]->target();
- }
+ TransitionTableIndex get_target(void)
+ {
+ return transitions[position]->target();
+ }
- SymbolNumber get_output(void)
- {
- return transitions[position]->get_output();
- }
+ SymbolNumber get_output(void)
+ {
+ return transitions[position]->get_output();
+ }
- SymbolNumber get_input(void)
- {
- return transitions[position]->get_input();
- }
+ SymbolNumber get_input(void)
+ {
+ return transitions[position]->get_input();
+ }
- bool get_finality(TransitionTableIndex i);
+ bool get_finality(TransitionTableIndex i);
- TransitionVector &operator() (void)
- {
- return transitions;
- }
+ TransitionVector &operator() (void)
+ {
+ return transitions;
+ }
};
class Transducer
{
- protected:
- TransducerHeader header;
- TransducerAlphabet alphabet;
- KeyTable * keys;
- IndexTableReader index_reader;
- TransitionTableReader transition_reader;
- Encoder encoder;
- DisplayVector display_vector;
-
- SymbolNumber * output_string;
-
- static const TransitionTableIndex START_INDEX = 0;
+protected:
+ TransducerHeader header;
+ TransducerAlphabet alphabet;
+ KeyTable * keys;
+ IndexTableReader index_reader;
+ TransitionTableReader transition_reader;
+ Encoder encoder;
+ DisplayVector display_vector;
+
+ SymbolNumber * output_string;
+
+ static const TransitionTableIndex START_INDEX = 0;
- std::vector<const char*> symbol_table;
+ std::vector<const char*> symbol_table;
- TransitionIndexVector &indices;
+ TransitionIndexVector &indices;
- TransitionVector &transitions;
+ TransitionVector &transitions;
- void set_symbol_table(void);
+ void set_symbol_table(void);
- virtual void note_analysis(SymbolNumber * whole_output_string);
+ virtual void note_analysis(SymbolNumber * whole_output_string);
- bool final_transition(TransitionTableIndex i)
- {
- return transitions[i]->final();
- }
+ bool final_transition(TransitionTableIndex i)
+ {
+ return transitions[i]->final();
+ }
- bool final_index(TransitionTableIndex i)
- {
- return indices[i]->final();
- }
+ bool final_index(TransitionTableIndex i)
+ {
+ return indices[i]->final();
+ }
- void try_epsilon_indices(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ void try_epsilon_indices(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- virtual void try_epsilon_transitions(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ virtual void try_epsilon_transitions(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- void find_index(SymbolNumber input,
- SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ void find_index(SymbolNumber input,
+ SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- void find_transitions(SymbolNumber input,
- SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ void find_transitions(SymbolNumber input,
+ SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- void get_analyses(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
-
-
- public:
- Transducer(FILE * f, TransducerHeader h, TransducerAlphabet a):
- header(h),
- alphabet(a),
- keys(alphabet.get_key_table()),
- index_reader(f,header.index_table_size()),
- transition_reader(f,header.target_table_size()),
- encoder(keys,header.input_symbol_count()),
- display_vector(),
- output_string((SymbolNumber*)(malloc(2000))),
- indices(index_reader()),
- transitions(transition_reader())
- {
- for (int i = 0; i < 1000; ++i)
- {
- output_string[i] = NO_SYMBOL_NUMBER;
- }
- set_symbol_table();
- }
+ void get_analyses(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
+
+
+public:
+ Transducer(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ header(h),
+ alphabet(a),
+ keys(alphabet.get_key_table()),
+ index_reader(f,header.index_table_size()),
+ transition_reader(f,header.target_table_size()),
+ encoder(keys,header.input_symbol_count()),
+ display_vector(),
+ output_string((SymbolNumber*)(malloc(2000))),
+ indices(index_reader()),
+ transitions(transition_reader())
+ {
+ for (int i = 0; i < 1000; ++i)
+ {
+ output_string[i] = NO_SYMBOL_NUMBER;
+ }
+ set_symbol_table();
+ }
- KeyTable * get_key_table(void)
- {
- return keys;
- }
-
- SymbolNumber find_next_key(char ** p)
- {
- return encoder.find_key(p);
- }
-
- void analyze(SymbolNumber * input_string)
- {
- get_analyses(input_string,output_string,output_string,START_INDEX);
- }
-
- virtual void printAnalyses(std::string prepend);
+ KeyTable * get_key_table(void)
+ {
+ return keys;
+ }
+
+ SymbolNumber find_next_key(char ** p)
+ {
+ return encoder.find_key(p);
+ }
+
+ void analyze(SymbolNumber * input_string)
+ {
+ get_analyses(input_string,output_string,output_string,START_INDEX);
+ }
+
+ virtual void printAnalyses(std::string prepend);
};
class TransducerUniq: public Transducer
{
- private:
- DisplaySet display_vector;
- void note_analysis(SymbolNumber * whole_output_string);
- public:
- TransducerUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
- Transducer(f, h, a),
- display_vector()
- {}
+private:
+ DisplaySet display_vector;
+ void note_analysis(SymbolNumber * whole_output_string);
+public:
+ TransducerUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ Transducer(f, h, a),
+ display_vector()
+ {}
- void printAnalyses(std::string prepend);
+ void printAnalyses(std::string prepend);
};
class TransducerFd: public Transducer
{
- FlagDiacriticStateStack statestack;
- OperationVector operations;
+ FlagDiacriticStateStack statestack;
+ OperationVector operations;
- void try_epsilon_transitions(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ void try_epsilon_transitions(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- bool PushState(FlagDiacriticOperation op);
-
- public:
- TransducerFd(FILE * f, TransducerHeader h, TransducerAlphabet a):
- Transducer(f, h, a),
- statestack(1, FlagDiacriticState (a.get_state_size(), 0)),
- operations(a.get_operation_vector())
- {}
+ bool PushState(FlagDiacriticOperation op);
+
+public:
+ TransducerFd(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ Transducer(f, h, a),
+ statestack(1, FlagDiacriticState (a.get_state_size(), 0)),
+ operations(a.get_operation_vector())
+ {}
};
class TransducerFdUniq: public TransducerFd
{
- private:
- DisplaySet display_vector;
- void note_analysis(SymbolNumber * whole_output_string);
- public:
- TransducerFdUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
- TransducerFd(f, h, a),
- display_vector()
- {}
+private:
+ DisplaySet display_vector;
+ void note_analysis(SymbolNumber * whole_output_string);
+public:
+ TransducerFdUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ TransducerFd(f, h, a),
+ display_vector()
+ {}
- void printAnalyses(std::string prepend);
+ void printAnalyses(std::string prepend);
};
@@ -688,377 +695,377 @@ typedef std::vector<TransitionW*> TransitionWVector;
class TransitionWIndex
{
- private:
- SymbolNumber input_symbol;
- TransitionTableIndex first_transition_index;
+private:
+ SymbolNumber input_symbol;
+ TransitionTableIndex first_transition_index;
- public:
+public:
- // Each TransitionIndex has an input symbol and a target index.
- static const size_t SIZE =
- sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
+ // Each TransitionIndex has an input symbol and a target index.
+ static const size_t SIZE =
+ sizeof(SymbolNumber) + sizeof(TransitionTableIndex);
- TransitionWIndex(SymbolNumber input,
- TransitionTableIndex first_transition):
- input_symbol(input),
- first_transition_index(first_transition)
- {}
+ TransitionWIndex(SymbolNumber input,
+ TransitionTableIndex first_transition):
+ input_symbol(input),
+ first_transition_index(first_transition)
+ {}
- bool matches(SymbolNumber s);
+ bool matches(SymbolNumber s);
- TransitionTableIndex target(void)
- {
- return first_transition_index;
- }
+ TransitionTableIndex target(void)
+ {
+ return first_transition_index;
+ }
- bool final(void)
- {
- return input_symbol == NO_SYMBOL_NUMBER &&
- first_transition_index != NO_TABLE_INDEX;
- }
+ bool final(void)
+ {
+ return input_symbol == NO_SYMBOL_NUMBER &&
+ first_transition_index != NO_TABLE_INDEX;
+ }
- Weight final_weight(void)
- {
- union to_weight
- {
- TransitionTableIndex i;
- Weight w;
- } weight;
- weight.i = first_transition_index;
- return weight.w;
- }
+ Weight final_weight(void)
+ {
+ union to_weight
+ {
+ TransitionTableIndex i;
+ Weight w;
+ } weight;
+ weight.i = first_transition_index;
+ return weight.w;
+ }
- SymbolNumber get_input(void)
- {
- return input_symbol;
- }
+ SymbolNumber get_input(void)
+ {
+ return input_symbol;
+ }
};
class TransitionW
{
- private:
- SymbolNumber input_symbol;
- SymbolNumber output_symbol;
- TransitionTableIndex target_index;
- Weight transition_weight;
-
- public:
-
- // Each transition has an input symbol an output symbol and
- // a target index, as well as a weight.
- static const size_t SIZE =
- 2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex) + sizeof(Weight);
-
- TransitionW(SymbolNumber input,
- SymbolNumber output,
- TransitionTableIndex target,
- Weight w):
- input_symbol(input),
- output_symbol(output),
- target_index(target),
- transition_weight(w)
- {}
-
- TransitionW():
- input_symbol(NO_SYMBOL_NUMBER),
- output_symbol(NO_SYMBOL_NUMBER),
- target_index(NO_TABLE_INDEX),
- transition_weight(INFINITE_WEIGHT)
- {}
-
- bool matches(SymbolNumber s);
-
- TransitionTableIndex target(void)
- {
- return target_index;
- }
-
- SymbolNumber get_output(void)
- {
- return output_symbol;
- }
-
- SymbolNumber get_input(void)
- {
- return input_symbol;
- }
+private:
+ SymbolNumber input_symbol;
+ SymbolNumber output_symbol;
+ TransitionTableIndex target_index;
+ Weight transition_weight;
+
+public:
+
+ // Each transition has an input symbol an output symbol and
+ // a target index, as well as a weight.
+ static const size_t SIZE =
+ 2 * sizeof(SymbolNumber) + sizeof(TransitionTableIndex) + sizeof(Weight);
+
+ TransitionW(SymbolNumber input,
+ SymbolNumber output,
+ TransitionTableIndex target,
+ Weight w):
+ input_symbol(input),
+ output_symbol(output),
+ target_index(target),
+ transition_weight(w)
+ {}
+
+ TransitionW():
+ input_symbol(NO_SYMBOL_NUMBER),
+ output_symbol(NO_SYMBOL_NUMBER),
+ target_index(NO_TABLE_INDEX),
+ transition_weight(INFINITE_WEIGHT)
+ {}
+
+ bool matches(SymbolNumber s);
+
+ TransitionTableIndex target(void)
+ {
+ return target_index;
+ }
+
+ SymbolNumber get_output(void)
+ {
+ return output_symbol;
+ }
+
+ SymbolNumber get_input(void)
+ {
+ return input_symbol;
+ }
- Weight get_weight(void)
- {
- return transition_weight;
- }
-
- bool final(void)
- {
- return input_symbol == NO_SYMBOL_NUMBER &&
- output_symbol == NO_SYMBOL_NUMBER &&
- target_index == 1;
- }
+ Weight get_weight(void)
+ {
+ return transition_weight;
+ }
+
+ bool final(void)
+ {
+ return input_symbol == NO_SYMBOL_NUMBER &&
+ output_symbol == NO_SYMBOL_NUMBER &&
+ target_index == 1;
+ }
};
class IndexTableReaderW
{
- private:
- TransitionTableIndex number_of_table_entries;
- char * TableIndices;
- TransitionWIndexVector indices;
- size_t table_size;
+private:
+ TransitionTableIndex number_of_table_entries;
+ char * TableIndices;
+ TransitionWIndexVector indices;
+ size_t table_size;
- void get_index_vector(void);
- public:
- IndexTableReaderW(FILE * f,
- TransitionTableIndex index_count):
- number_of_table_entries(index_count)
- {
- table_size = number_of_table_entries*TransitionWIndex::SIZE;
- TableIndices = (char*)(malloc(table_size));
-
- // This dummy variable is needed, since the compiler complains
- // for not catching the return value of fread().
- int dummy_number_of_bytes;
-
- dummy_number_of_bytes = fread(TableIndices,table_size,1,f);
- (void)dummy_number_of_bytes;
- get_index_vector();
- }
+ void get_index_vector(void);
+public:
+ IndexTableReaderW(FILE * f,
+ TransitionTableIndex index_count):
+ number_of_table_entries(index_count)
+ {
+ table_size = number_of_table_entries*TransitionWIndex::SIZE;
+ TableIndices = (char*)(malloc(table_size));
+
+ // This dummy variable is needed, since the compiler complains
+ // for not catching the return value of fread().
+ int dummy_number_of_bytes;
+
+ dummy_number_of_bytes = fread(TableIndices,table_size,1,f);
+ (void)dummy_number_of_bytes;
+ get_index_vector();
+ }
- bool get_finality(TransitionTableIndex i)
- {
- return indices[i]->final();
- }
+ bool get_finality(TransitionTableIndex i)
+ {
+ return indices[i]->final();
+ }
- TransitionWIndex * at(TransitionTableIndex i)
- {
- return indices[i];
- }
+ TransitionWIndex * at(TransitionTableIndex i)
+ {
+ return indices[i];
+ }
- TransitionWIndexVector &operator() (void)
- { return indices; }
+ TransitionWIndexVector &operator() (void)
+ { return indices; }
};
class TransitionTableReaderW
{
- private:
- TransitionTableIndex number_of_table_entries;
- char * TableTransitions;
- TransitionWVector transitions;
- size_t table_size;
+private:
+ TransitionTableIndex number_of_table_entries;
+ char * TableTransitions;
+ TransitionWVector transitions;
+ size_t table_size;
- TransitionTableIndex position;
+ TransitionTableIndex position;
- void get_transition_vector(void);
-
- public:
- TransitionTableReaderW(FILE * f,
- TransitionTableIndex transition_count):
- number_of_table_entries(transition_count),
- position(0)
- {
- table_size = number_of_table_entries*TransitionW::SIZE;
- TableTransitions = (char*)(malloc(table_size));
- int bytes;
- bytes = fread(TableTransitions,table_size,1,f);
- (void)bytes;
- get_transition_vector();
- }
+ void get_transition_vector(void);
+
+public:
+ TransitionTableReaderW(FILE * f,
+ TransitionTableIndex transition_count):
+ number_of_table_entries(transition_count),
+ position(0)
+ {
+ table_size = number_of_table_entries*TransitionW::SIZE;
+ TableTransitions = (char*)(malloc(table_size));
+ int bytes;
+ bytes = fread(TableTransitions,table_size,1,f);
+ (void)bytes;
+ get_transition_vector();
+ }
- void Set(TransitionTableIndex pos);
+ void Set(TransitionTableIndex pos);
- TransitionW * at(TransitionTableIndex i)
- {
- return transitions[i - TRANSITION_TARGET_TABLE_START];
- }
+ TransitionW * at(TransitionTableIndex i)
+ {
+ return transitions[i - TRANSITION_TARGET_TABLE_START];
+ }
- void Next(void)
- {
- ++position;
- }
+ void Next(void)
+ {
+ ++position;
+ }
- bool Matches(SymbolNumber s);
+ bool Matches(SymbolNumber s);
- TransitionTableIndex get_target(void)
- {
- return transitions[position]->target();
- }
+ TransitionTableIndex get_target(void)
+ {
+ return transitions[position]->target();
+ }
- SymbolNumber get_output(void)
- {
- return transitions[position]->get_output();
- }
+ SymbolNumber get_output(void)
+ {
+ return transitions[position]->get_output();
+ }
- SymbolNumber get_input(void)
- {
- return transitions[position]->get_input();
- }
+ SymbolNumber get_input(void)
+ {
+ return transitions[position]->get_input();
+ }
- bool get_finality(TransitionTableIndex i);
+ bool get_finality(TransitionTableIndex i);
- TransitionWVector &operator() (void)
- {
- return transitions;
- }
+ TransitionWVector &operator() (void)
+ {
+ return transitions;
+ }
};
class TransducerW
{
- protected:
+protected:
- TransducerHeader header;
- TransducerAlphabet alphabet;
- KeyTable * keys;
- IndexTableReaderW index_reader;
- TransitionTableReaderW transition_reader;
- Encoder encoder;
- DisplayMultiMap display_map;
+ TransducerHeader header;
+ TransducerAlphabet alphabet;
+ KeyTable * keys;
+ IndexTableReaderW index_reader;
+ TransitionTableReaderW transition_reader;
+ Encoder encoder;
+ DisplayMultiMap display_map;
- SymbolNumber * output_string;
+ SymbolNumber * output_string;
- static const TransitionTableIndex START_INDEX = 0;
+ static const TransitionTableIndex START_INDEX = 0;
- std::vector<const char*> symbol_table;
+ std::vector<const char*> symbol_table;
- TransitionWIndexVector &indices;
+ TransitionWIndexVector &indices;
- TransitionWVector &transitions;
+ TransitionWVector &transitions;
- Weight current_weight;
+ Weight current_weight;
- void set_symbol_table(void);
+ void set_symbol_table(void);
- virtual void try_epsilon_transitions(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ virtual void try_epsilon_transitions(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- void try_epsilon_indices(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
-
- void find_transitions(SymbolNumber input,
- SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
-
- void find_index(SymbolNumber input,
- SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
-
- virtual void note_analysis(SymbolNumber * whole_output_string);
-
- bool final_transition(TransitionTableIndex i)
- {
- return transitions[i]->final();
- }
+ void try_epsilon_indices(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
+
+ void find_transitions(SymbolNumber input,
+ SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
+
+ void find_index(SymbolNumber input,
+ SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
+
+ virtual void note_analysis(SymbolNumber * whole_output_string);
+
+ bool final_transition(TransitionTableIndex i)
+ {
+ return transitions[i]->final();
+ }
- bool final_index(TransitionTableIndex i)
- {
- return indices[i]->final();
- }
-
- void get_analyses(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
-
- Weight get_final_index_weight(TransitionTableIndex i) {
- return indices[i]->final_weight();
- }
-
- Weight get_final_transition_weight(TransitionTableIndex i) {
- return transitions[i]->get_weight();
- }
-
- public:
- TransducerW(FILE * f, TransducerHeader h, TransducerAlphabet a) :
- header(h),
- alphabet(a),
- keys(alphabet.get_key_table()),
- index_reader(f,header.index_table_size()),
- transition_reader(f,header.target_table_size()),
- encoder(keys,header.input_symbol_count()),
- display_map(),
- output_string((SymbolNumber*)(malloc(2000))),
- indices(index_reader()),
- transitions(transition_reader()),
- current_weight(0.0)
- {
- for (int i = 0; i < 1000; ++i)
- {
- output_string[i] = NO_SYMBOL_NUMBER;
- }
- set_symbol_table();
- }
-
- KeyTable * get_key_table(void)
- {
- return keys;
- }
-
- void analyze(SymbolNumber * input_string)
- {
- get_analyses(input_string,output_string,output_string,START_INDEX);
- }
-
-
- SymbolNumber find_next_key(char ** p)
- {
- return encoder.find_key(p);
- }
-
- virtual void printAnalyses(std::string prepend);
+ bool final_index(TransitionTableIndex i)
+ {
+ return indices[i]->final();
+ }
+
+ void get_analyses(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
+
+ Weight get_final_index_weight(TransitionTableIndex i) {
+ return indices[i]->final_weight();
+ }
+
+ Weight get_final_transition_weight(TransitionTableIndex i) {
+ return transitions[i]->get_weight();
+ }
+
+public:
+ TransducerW(FILE * f, TransducerHeader h, TransducerAlphabet a) :
+ header(h),
+ alphabet(a),
+ keys(alphabet.get_key_table()),
+ index_reader(f,header.index_table_size()),
+ transition_reader(f,header.target_table_size()),
+ encoder(keys,header.input_symbol_count()),
+ display_map(),
+ output_string((SymbolNumber*)(malloc(2000))),
+ indices(index_reader()),
+ transitions(transition_reader()),
+ current_weight(0.0)
+ {
+ for (int i = 0; i < 1000; ++i)
+ {
+ output_string[i] = NO_SYMBOL_NUMBER;
+ }
+ set_symbol_table();
+ }
+
+ KeyTable * get_key_table(void)
+ {
+ return keys;
+ }
+
+ void analyze(SymbolNumber * input_string)
+ {
+ get_analyses(input_string,output_string,output_string,START_INDEX);
+ }
+
+
+ SymbolNumber find_next_key(char ** p)
+ {
+ return encoder.find_key(p);
+ }
+
+ virtual void printAnalyses(std::string prepend);
};
class TransducerWUniq: public TransducerW
{
- private:
- DisplayMap display_map;
- void note_analysis(SymbolNumber * whole_output_string);
- public:
- TransducerWUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
- TransducerW(f, h, a),
- display_map()
- {}
+private:
+ DisplayMap display_map;
+ void note_analysis(SymbolNumber * whole_output_string);
+public:
+ TransducerWUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ TransducerW(f, h, a),
+ display_map()
+ {}
- void printAnalyses(std::string prepend);
+ void printAnalyses(std::string prepend);
};
class TransducerWFd: public TransducerW
{
- FlagDiacriticStateStack statestack;
- OperationVector operations;
+ FlagDiacriticStateStack statestack;
+ OperationVector operations;
- void try_epsilon_transitions(SymbolNumber * input_symbol,
- SymbolNumber * output_symbol,
- SymbolNumber * original_output_string,
- TransitionTableIndex i);
+ void try_epsilon_transitions(SymbolNumber * input_symbol,
+ SymbolNumber * output_symbol,
+ SymbolNumber * original_output_string,
+ TransitionTableIndex i);
- bool PushState(FlagDiacriticOperation op);
+ bool PushState(FlagDiacriticOperation op);
- public:
- TransducerWFd(FILE * f, TransducerHeader h, TransducerAlphabet a):
- TransducerW(f, h, a),
- statestack(1, FlagDiacriticState (a.get_state_size(), 0)),
- operations(a.get_operation_vector())
- {}
+public:
+ TransducerWFd(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ TransducerW(f, h, a),
+ statestack(1, FlagDiacriticState (a.get_state_size(), 0)),
+ operations(a.get_operation_vector())
+ {}
};
class TransducerWFdUniq: public TransducerWFd
{
- private:
- DisplayMap display_map;
- void note_analysis(SymbolNumber * whole_output_string);
- public:
- TransducerWFdUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
- TransducerWFd(f, h, a),
- display_map()
- {}
+private:
+ DisplayMap display_map;
+ void note_analysis(SymbolNumber * whole_output_string);
+public:
+ TransducerWFdUniq(FILE * f, TransducerHeader h, TransducerAlphabet a):
+ TransducerWFd(f, h, a),
+ display_map()
+ {}
- void printAnalyses(std::string prepend);
+ void printAnalyses(std::string prepend);
};
diff --git a/tools/src/hfst-pair-test.cc b/tools/src/hfst-pair-test.cc
index 24e54ff..8ed198b 100644
--- a/tools/src/hfst-pair-test.cc
+++ b/tools/src/hfst-pair-test.cc
@@ -52,6 +52,7 @@ static FILE* pair_test_file;
static size_t linen = 0;
static bool pair_test_given = false;
static bool positive_test = true;
+static bool xerox_mode = false;
using hfst::HfstInputStream;
using hfst::HfstTransducer;
@@ -80,8 +81,10 @@ print_usage()
"Input/Output options:\n"
" -i, --input=INFILE Read input rule file from INFILE\n"
" -o, --output=OUTFILE Write test output to OUTFILE\n"
- " -N --negative-test Test fails if any of the pair strings is\n"
- " accepted.\n");
+ " -N --negative-test Test fails if any of the pair strings is\n"
+ " accepted.\n"
+ " -X --xerox-mode In xerox mode, test cases are harvested\n"
+ " from a twolc source file.\n");
fprintf(message_out, "Pair test options:\n"
" -I, --input-strings=SFILE Read pair test strings from\n"
@@ -104,13 +107,15 @@ print_usage()
"considered comment lines and skipped.\n");
fprintf(message_out, "\n");
fprintf(message_out,
- "There are two test modes positive and negative. In positive\n"
- "mode, all of the pair strings should be allowed and in negative\n"
- "mode they should be disallowed.\n"
+ "There are three test modes positive, negative and Xerox mode. In\n"
+ "positive mode, all of the pair strings should be allowed and in\n"
+ "negative mode they should be disallowed. In Xerox mode the cases\n"
+ "are read from a twolc source file and both positive and negative\n"
+ "cases can occur.\n"
);
fprintf(message_out, "\n");
fprintf(message_out,
- "Ordinarily, positive test mode is in use. Option -N switches to\n"
+ "Ordinarily, positive test mode is in use. Option -N switches to\n"
"negative test mode. The exit code for a successful test is 0. \n"
"The exit code is 1 otherwise. A successful test will print\n"
"\"Test passed\". A failing test prints \"Test failed\" and\n"
@@ -124,6 +129,21 @@ print_usage()
"the strings that are allowed are printed.\n");
fprintf(message_out, "\n");
fprintf(message_out,
+ "In Xerox mode, the input should be a twolc file. Tests consist of\n"
+ "two lines: an input form and an output form. The test cases are\n"
+ "specialized comments prefixed with either '!€' or '!$' depeding on\n"
+ "whether the pair should succeed or fail. An example of a positive\n"
+ "test:\n\n"
+
+ "!€ earlYer\n"
+ "!€ earlier\n\n"
+
+ "An example of a negative test:\n\n"
+
+ "!$ earlYer\n"
+ "!$ earlyer\n");
+ fprintf(message_out, "\n");
+ fprintf(message_out,
"In silent mode (-s), the program won't print anything. Only the\n"
"exit code tells whether the test was successful or not.\n");
fprintf(message_out, "\n");
@@ -146,12 +166,13 @@ parse_options(int argc, char** argv)
// add tool-specific options here
{"input-strings", required_argument, 0, 'I'},
{"negative-test", no_argument, 0, 'N'},
+ {"xerox-mode", no_argument, 0, 'X'},
{0,0,0,0}
};
int option_index = 0;
// add tool-specific options here
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "I:N",
+ HFST_GETOPT_UNARY_SHORT "I:Nx",
long_options, &option_index);
if (-1 == c)
{
@@ -171,6 +192,9 @@ parse_options(int argc, char** argv)
case 'N':
positive_test = false;
break;
+ case 'X':
+ xerox_mode = true;
+ break;
#include "inc/getopt-cases-error.h"
}
}
@@ -429,6 +453,25 @@ void get_symbols(HfstBasicTransducer &t,SymbolSet &known_symbols)
}
}
+std::string strip_space(const std::string &line)
+{
+ int first_non_white_space_pos = line.find_first_not_of(" \t");
+
+ if (first_non_white_space_pos == std::string::npos)
+ { return ""; }
+
+ int last_non_white_space_pos = line.find_last_not_of(" \t");
+ int len = last_non_white_space_pos - first_non_white_space_pos + 1;
+
+ return line.substr(first_non_white_space_pos, len);
+}
+
+bool is_positive_test_line(const std::string &line)
+{ return strip_space(line).substr(0,sizeof("!€") - 1) == "!€"; }
+
+bool is_negative_test_line(const std::string &line)
+{ return strip_space(line).substr(0,sizeof("!$") - 1) == "!$"; }
+
int
process_stream(HfstInputStream& inputstream, FILE* outstream)
{
@@ -471,65 +514,184 @@ process_stream(HfstInputStream& inputstream, FILE* outstream)
char* line = 0;
size_t llen = 0;
- // Define tokenizer with no multi character symbols and an
- // empty epsilon representation.
- StringVector empty_v;
- HfstStrings2FstTokenizer input_tokenizer
- (empty_v, std::string("0"));
int exit_code = 0;
- while (hfst_getline(&line, &llen, pair_test_file) != -1)
+
+ if (not xerox_mode)
{
- linen++;
- char *p = line;
- while (*p != '\0')
+ // Define tokenizer with no multi character symbols and an
+ // empty epsilon representation.
+ StringVector empty_v;
+ hfst::HfstStrings2FstTokenizer input_tokenizer
+ (empty_v, std::string("0"));
+
+ while (hfst_getline(&line, &llen, pair_test_file) != -1)
{
- if (*p == '\n')
+ linen++;
+ char *p = line;
+ while (*p != '\0')
{
- *p = '\0';
- break;
+ if (*p == '\n')
+ {
+ *p = '\0';
+ break;
+ }
+ p++;
}
- p++;
- }
- if (is_empty_or_comment(line))
- { continue; }
- verbose_printf("Pair test on %s...\n", line);
-
- int new_exit_code = 0;
-
- try
- {
- StringPairVector tokenized_pair_string =
- input_tokenizer.tokenize_pair_string(line,true);
-
- tokenized_pair_string.insert
- (tokenized_pair_string.begin(),
- StringPair("@#@",hfst::internal_epsilon));
- tokenized_pair_string.insert
- (tokenized_pair_string.end(),
- StringPair("@#@",hfst::internal_epsilon));
-
- new_exit_code =
- test(tokenized_pair_string,line,grammar,rule_names,
- positive_test,outfile,known_symbols);
-
+ if (is_empty_or_comment(line))
+ { continue; }
+ verbose_printf("Pair test on %s...\n", line);
+
+ int new_exit_code = 0;
+
+ try
+ {
+ StringPairVector tokenized_pair_string =
+ input_tokenizer.tokenize_pair_string(line,true);
+
+ tokenized_pair_string.insert
+ (tokenized_pair_string.begin(),
+ StringPair("@#@",hfst::internal_epsilon));
+ tokenized_pair_string.insert
+ (tokenized_pair_string.end(),
+ StringPair("@#@",hfst::internal_epsilon));
+
+ new_exit_code =
+ test(tokenized_pair_string,line,grammar,rule_names,
+ positive_test,outfile,known_symbols);
+
+ }
+ catch (const hfst::UnescapedColsFound &e)
+ {
+ error(EXIT_FAILURE, 0,
+ "The correspondence %s contains unquoted colon-symbols. If "
+ "you want to input pairs where either symbol is epsilon, "
+ "use 0 e.g. \"m a s s 0:e s\".\n",
+ line);
+
+ }
+
+
+ if (exit_code == 0)
+ { exit_code = new_exit_code; }
+
+ } // while lines in input
+ free(line);
}
- catch (const UnescapedColsFound &e)
+ else
{
- error(EXIT_FAILURE, 0,
- "The correspondence %s contains unquoted colon-symbols. If "
- "you want to input pairs where either symbol is epsilon, "
- "use 0 e.g. \"m a s s 0:e s\".\n",
- line);
+ /*
+ Read test cases from a twolc source file.
+
+ Positive test cases are prefixed by "!€" and negative test
+ cases by "!$".
+
+ Each test case spans two lines: the input and output cases.
+ */
+
+ StringVector positive_test_cases;
+ StringVector negative_test_cases;
+
+ StringVector symbols(known_symbols.begin(), known_symbols.end());
+
+ hfst::HfstStrings2FstTokenizer input_tokenizer
+ (symbols, std::string("0"));
+
+ while (hfst_getline(&line, &llen, pair_test_file) != -1)
+ {
+ linen++;
+ char *p = line;
+ while (*p != '\0')
+ {
+ if (*p == '\n')
+ {
+ *p = '\0';
+ break;
+ }
+ p++;
+ }
+
+ if (is_positive_test_line(line))
+ {
+ // "!€ xyz" -> "xyz"
+ std::string test_case = strip_space(line).substr(sizeof("!€") - 1);
+ test_case = strip_space(test_case);
+
+ positive_test_cases.push_back(test_case);
+ verbose_printf("Positive test case: %s...\n",
+ test_case.c_str());
+ }
+ else if (is_negative_test_line(line))
+ {
+ // "!$ xyz" -> "xyz"
+ std::string test_case = strip_space(line).substr(sizeof("!$") - 1);
+ test_case = strip_space(test_case);
+
+ negative_test_cases.push_back(test_case);
+ verbose_printf("Negative test case: %s %s...\n",
+ line, test_case.c_str());
+ }
+ else
+ { continue; }
+
+
+ } // while lines in input
+ free(line);
- }
+ if (positive_test_cases.size() % 2 != 0)
+ {
+ error(EXIT_FAILURE, 0,
+ "Got an odd number of positive test cases. Every input string\n"
+ "has to have an output string.\n");
+ }
-
- if (exit_code == 0)
- { exit_code = new_exit_code; }
+ if (negative_test_cases.size() % 2 != 0)
+ {
+ error(EXIT_FAILURE, 0,
+ "Got an odd number of negative test cases. Every input string\n"
+ "has to have an output string.\n");
+ }
- } // while lines in input
- free(line);
+ for (int i = 0; i < positive_test_cases.size(); i += 2)
+ {
+ const std::string &input_case = positive_test_cases[i];
+ const std::string &output_case = positive_test_cases[i + 1];
+
+ StringPairVector test_case = input_tokenizer.tokenize_string_pair
+ (input_case + ":" + output_case, false);
+
+ int new_exit_code = test(test_case,
+ input_case + " : " + output_case,
+ grammar,
+ rule_names,
+ true,
+ outfile,
+ known_symbols);
+
+ if (exit_code == 0)
+ { exit_code = new_exit_code; }
+ }
+
+ for (int i = 0; i < negative_test_cases.size(); i += 2)
+ {
+ const std::string &input_case = negative_test_cases[i];
+ const std::string &output_case = negative_test_cases[i + 1];
+
+ StringPairVector test_case = input_tokenizer.tokenize_string_pair
+ (input_case + ":" + output_case, false);
+
+ int new_exit_code = test(test_case,
+ input_case + " : " + output_case,
+ grammar,
+ rule_names,
+ false,
+ outfile,
+ known_symbols);
+
+ if (exit_code == 0)
+ { exit_code = new_exit_code; }
+ }
+ }
return exit_code;
}
diff --git a/tools/src/hfst-pmatch.cc b/tools/src/hfst-pmatch.cc
index d9e3d1d..6655e91 100644
--- a/tools/src/hfst-pmatch.cc
+++ b/tools/src/hfst-pmatch.cc
@@ -36,7 +36,14 @@ using std::pair;
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+# include "hfst-string-conversions.h"
+#else
+# include <getopt.h>
+#endif
+
#include <math.h>
#include <errno.h>
@@ -87,25 +94,48 @@ void match_and_print(hfst_ol::PmatchContainer & container,
input_text.erase(input_text.size() -1, 1);
}
if (!locate_mode) {
- outstream << container.match(input_text);
+#ifndef _MSC_VER
+ outstream << container.match(input_text);
+#else
+ hfst::hfst_fprintf(stdout, "%s", container.match(input_text).c_str());
+#endif
} else {
hfst_ol::LocationVectorVector locations = container.locate(input_text);
for(hfst_ol::LocationVectorVector::const_iterator it = locations.begin();
it != locations.end(); ++it) {
- outstream << it->at(0).start << "|" << it->at(0).length << "|"
- << it->at(0).output << "|" << it->at(0).tag << std::endl;
+ if (it->at(0).output.compare("@_NONMATCHING_@") != 0) {
+#ifndef _MSC_VER
+ outstream << it->at(0).start << "|" << it->at(0).length << "|"
+ << it->at(0).output << "|" << it->at(0).tag << std::endl;
+#else
+ hfst::hfst_fprintf(stdout, "%i|%i|%s|%s\n", it->at(0).start, it->at(0).length, it->at(0).output.c_str(), it->at(0).tag.c_str());
+#endif
+ }
}
}
outstream << std::endl;
}
+
int process_input(hfst_ol::PmatchContainer & container,
std::ostream & outstream)
{
std::string input_text;
char * line = NULL;
size_t len = 0;
- while (hfst_getline(&line, &len, inputfile) > 0) {
+ while (true) {
+
+#ifndef _MSC_VER
+ if (!(hfst_getline(&line, &len, inputfile) > 0))
+ break;
+#else
+ std::string linestr("");
+ size_t bufsize = 1000;
+ if (! hfst::get_line_from_console(linestr, bufsize, true /* keep newlines */))
+ break;
+ line = strdup(linestr.c_str());
+#endif
+
if (!blankline_separated) {
// newline separated
input_text = line;
@@ -146,7 +176,7 @@ int parse_options(int argc, char** argv)
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nxl",
+ char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nxlp",
long_options, &option_index);
if (-1 == c)
{
@@ -223,8 +253,7 @@ int main(int argc, char ** argv)
}
hfst_ol::PmatchContainer container(instream);
container.set_verbose(verbose);
-// the locate_mode bool in this tool only affects its own processing
- container.set_locate_mode(extract_tags);
+ container.set_extract_tags_mode(extract_tags);
container.set_profile(profile);
// if (outfile != stdout) {
// std::filebuf fb;
@@ -233,5 +262,10 @@ int main(int argc, char ** argv)
// return process_input(container, outstream);
// fb.close();
// } else {
+
+#ifdef _MSC_VER
+ hfst::print_output_to_console(true);
+#endif
+
return process_input(container, std::cout);
}
diff --git a/tools/src/hfst-pmatch2fst.cc b/tools/src/hfst-pmatch2fst.cc
index 00a128a..c87f609 100644
--- a/tools/src/hfst-pmatch2fst.cc
+++ b/tools/src/hfst-pmatch2fst.cc
@@ -41,7 +41,13 @@ using std::pair;
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
+
#include <math.h>
#include <errno.h>
#include <time.h>
@@ -162,6 +168,15 @@ process_stream(HfstOutputStream& outstream)
std::string file_contents;
std::map<std::string, HfstTransducer*> definitions;
int c;
+
+#ifdef _MSC_VER
+ if (inputfile == stdin && !silent)
+ {
+ warning(0, 0, "Reading from standard input. UTF-8 characters\n"
+ "outside ascii range are supported only if input comes from a file.");
+ }
+#endif
+
while ((c = fgetc(inputfile)) != EOF) {
file_contents.push_back(c);
}
@@ -175,97 +190,123 @@ process_stream(HfstOutputStream& outstream)
std::cerr << "Building hfst-ol alphabet... ";
}
- // First we need to collect a unified alphabet from all the transducers.
- std::string unified_alphabet;
- hfst::StringSet symbols_seen;
- hfst::HfstTokenizer tok;
- for (std::map<std::string, HfstTransducer *>::const_iterator it =
- definitions.begin(); it != definitions.end(); ++it) {
- hfst::StringSet string_set = it->second->get_alphabet();
- for (hfst::StringSet::const_iterator sym = string_set.begin();
- sym != string_set.end(); ++sym) {
- if (symbols_seen.count(*sym) == 0) {
- unified_alphabet.append(*sym);
- tok.add_multichar_symbol(*sym);
- symbols_seen.insert(*sym);
+ if (definitions.size() > 1) {
+
+ // A dummy transducer with an alphabet with all the symbols
+ HfstTransducer harmonizer(compilation_format);
+
+ // First we need to collect a unified alphabet from all the transducers.
+ hfst::StringSet symbols_seen;
+ for (std::map<std::string, HfstTransducer *>::const_iterator it =
+ definitions.begin(); it != definitions.end(); ++it) {
+ hfst::StringSet string_set = it->second->get_alphabet();
+ for (hfst::StringSet::const_iterator sym = string_set.begin();
+ sym != string_set.end(); ++sym) {
+ if (symbols_seen.count(*sym) == 0) {
+ harmonizer.disjunct(HfstTransducer(*sym, compilation_format));
+ symbols_seen.insert(*sym);
+ }
}
}
- }
- if (unified_alphabet.empty()) {
- // We don't recognise anything, go home early
- std::cerr << program_name << ": Empty ruleset, nothing to write\n";
- return EXIT_FAILURE;
- }
- // Now we make a dummy transducer with that alphabet
- HfstTransducer dummy(unified_alphabet, tok, compilation_format);
-
- // Then we convert it...
- HfstTransducer * harmonizer = &dummy.convert(hfst::HFST_OLW_TYPE);
- // Use these for naughty intermediate steps to make sure
- // everything has the same alphabet
- hfst::HfstBasicTransducer * intermediate_tmp;
- hfst_ol::Transducer * harmonized_tmp;
- hfst::HfstTransducer * output_tmp;
-
- if (verbose) {
- double duration = (clock() - timer) /
- (double) CLOCKS_PER_SEC;
- timer = clock();
- std::cerr << "built in " << duration << " seconds\n";
- std::cerr << "Converting TOP... ";
- }
-
+ if (symbols_seen.size() == 0) {
+ // We don't recognise anything, go home early
+ std::cerr << program_name << ": Empty ruleset, nothing to write\n";
+ return EXIT_FAILURE;
+ }
+
+ // Then we convert it...
+ harmonizer.convert(hfst::HFST_OLW_TYPE);
+ // Use these for naughty intermediate steps to make sure
+ // everything has the same alphabet
+ hfst::HfstBasicTransducer * intermediate_tmp;
+ hfst_ol::Transducer * harmonized_tmp;
+ hfst::HfstTransducer * output_tmp;
+
+ if (verbose) {
+ double duration = (clock() - timer) /
+ (double) CLOCKS_PER_SEC;
+ timer = clock();
+ std::cerr << "built in " << duration << " seconds\n";
+ std::cerr << "Converting TOP... ";
+ }
+
// When done compiling everything, look for TOP and output it first.
- if (definitions.count("TOP") == 1) {
- intermediate_tmp = hfst::implementations::ConversionFunctions::
- hfst_transducer_to_hfst_basic_transducer(*definitions["TOP"]);
- harmonized_tmp = hfst::implementations::ConversionFunctions::
- hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
- true, // weighted
- "", // no special options
- harmonizer); // harmonize with this
- output_tmp = hfst::implementations::ConversionFunctions::
- hfst_ol_to_hfst_transducer(harmonized_tmp);
- output_tmp->set_name("TOP");
- outstream << *output_tmp;
- delete definitions["TOP"];
- definitions.erase("TOP");
- delete intermediate_tmp;
- delete output_tmp;
- }
- if (verbose) {
- double duration = (clock() - timer) /
- (double) CLOCKS_PER_SEC;
- timer = clock();
- std::cerr << "converted in " << duration << " seconds\n";
- }
-
- for (std::map<std::string, HfstTransducer *>::iterator it =
- definitions.begin(); it != definitions.end(); ++it) {
+ if (definitions.count("TOP") == 1) {
+ intermediate_tmp = hfst::implementations::ConversionFunctions::
+ hfst_transducer_to_hfst_basic_transducer(*definitions["TOP"]);
+ harmonized_tmp = hfst::implementations::ConversionFunctions::
+ hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
+ true, // weighted
+ "", // no special options
+ &harmonizer); // harmonize with this
+ output_tmp = hfst::implementations::ConversionFunctions::
+ hfst_ol_to_hfst_transducer(harmonized_tmp);
+ output_tmp->set_name("TOP");
+ outstream << *output_tmp;
+ delete definitions["TOP"];
+ definitions.erase("TOP");
+ delete intermediate_tmp;
+ delete output_tmp;
+ }
if (verbose) {
- std::cerr << "Converting " << it->first << "... ";
+ double duration = (clock() - timer) /
+ (double) CLOCKS_PER_SEC;
timer = clock();
+ std::cerr << "converted in " << duration << " seconds\n";
+ }
+
+ for (std::map<std::string, HfstTransducer *>::iterator it =
+ definitions.begin(); it != definitions.end(); ++it) {
+ if (verbose) {
+ std::cerr << "Converting " << it->first << "... ";
+ timer = clock();
}
- intermediate_tmp = hfst::implementations::ConversionFunctions::
+ intermediate_tmp = hfst::implementations::ConversionFunctions::
hfst_transducer_to_hfst_basic_transducer(*(it->second));
- harmonized_tmp = hfst::implementations::ConversionFunctions::
- hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
- true, // weighted
- "", // no special options
- harmonizer); // harmonize with this
- output_tmp = hfst::implementations::ConversionFunctions::
- hfst_ol_to_hfst_transducer(harmonized_tmp);
- output_tmp->set_name(it->first);
- outstream << *output_tmp;;
- delete it->second;
- delete intermediate_tmp;
- delete output_tmp;
+ harmonized_tmp = hfst::implementations::ConversionFunctions::
+ hfst_basic_transducer_to_hfst_ol(intermediate_tmp,
+ true, // weighted
+ "", // no special options
+ &harmonizer); // harmonize with this
+ output_tmp = hfst::implementations::ConversionFunctions::
+ hfst_ol_to_hfst_transducer(harmonized_tmp);
+ output_tmp->set_name(it->first);
+ outstream << *output_tmp;;
+ delete it->second;
+ delete intermediate_tmp;
+ delete output_tmp;
+ if (verbose) {
+ double duration = (clock() - timer) /
+ (double) CLOCKS_PER_SEC;
+ std::cerr << "converted in " << duration << " seconds\n";
+ }
+ }
+ } else if (definitions.size() == 1) {
+ // A common case of only one top-level definition
if (verbose) {
double duration = (clock() - timer) /
(double) CLOCKS_PER_SEC;
+ timer = clock();
+ std::cerr << "built in " << duration << " seconds\n";
+ std::cerr << "Converting TOP... ";
+ }
+
+ std::map<std::string, HfstTransducer *>::iterator only_entry = definitions.begin();
+ only_entry->second->convert(hfst::HFST_OLW_TYPE);
+
+ if (verbose) {
+ double duration = (clock() - timer) /
+ (double) CLOCKS_PER_SEC;
+ timer = clock();
std::cerr << "converted in " << duration << " seconds\n";
}
+
+ outstream << *(only_entry->second);
+ } else {
+ std::cerr << program_name << ": Empty ruleset, nothing to write\n";
+ return EXIT_FAILURE;
}
+
return EXIT_SUCCESS;
}
diff --git a/tools/src/hfst-proc/alphabet.cc b/tools/src/hfst-proc/alphabet.cc
index f381def..14dca41 100644
--- a/tools/src/hfst-proc/alphabet.cc
+++ b/tools/src/hfst-proc/alphabet.cc
@@ -242,7 +242,7 @@ ProcTransducerAlphabet::check_for_overlapping() const
if(!overlapping.empty())
{
- if (not silentFlag)
+ if (! silentFlag)
{
std::cerr << "!! Warning: Transducer contains one or more multi-character symbols made up of\n"
<< "ASCII characters which are also available as single-character symbols. The\n"
diff --git a/tools/src/hfst-proc/hfst-proc.cc b/tools/src/hfst-proc/hfst-proc.cc
index 8cce730..7e112b9 100644
--- a/tools/src/hfst-proc/hfst-proc.cc
+++ b/tools/src/hfst-proc/hfst-proc.cc
@@ -10,7 +10,16 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
-#include <getopt.h>
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+# include "../hfst-string-conversions.h"
+#else
+# include <getopt.h>
+#endif
+
+#include <cstdio>
+#include "../inc/globals-common.h"
+
#include <fstream>
#include <cstdlib>
#include "hfst-proc.h"
@@ -58,7 +67,7 @@ static bool handle_hfst3_header(std::istream& is)
if (!strcmp(headervalue, "type")) {
is.getline(headervalue, remaining_header_len + 1, '\0');
remaining_header_len -= strlen(headervalue) + 1;
- if (strcmp(headervalue, "HFST_OL") and
+ if (strcmp(headervalue, "HFST_OL") &&
strcmp(headervalue, "HFST_OLW")) {
delete headervalue;
HFST_THROW(TransducerHasWrongTypeException);
@@ -129,7 +138,9 @@ bool print_usage(void)
" -X, --raw Do not perform any mangling to:\n"
" case, ``superblanks'' or anything else!!!\n"
"\n" <<
+#ifdef HAVE_CONFIG_H
"Report bugs to " << PACKAGE_BUGREPORT << "\n" <<
+#endif
"\n";
return true;
}
@@ -138,8 +149,11 @@ bool print_version(void)
{
std::cout <<
"\n" <<
- "hfst-proc 0.0 (" <<
- PACKAGE_STRING << ")" << std::endl <<
+ "hfst-proc 0.0" <<
+#ifdef HAVE_CONFIG_H
+ " (" << PACKAGE_STRING << ")" <<
+#endif
+ std::endl <<
__DATE__ << " " __TIME__ << std::endl <<
"copyright (C) 2009-2011 University of Helsinki\n";
return true;
@@ -344,6 +358,17 @@ int main(int argc, char **argv)
return EXIT_FAILURE;
}
}
+#ifdef _MSC_VER
+ else
+ {
+ hfst::set_console_cp_to_utf8();
+ if (!silentFlag)
+ {
+ std::cerr << "hfst-proc: warning: Reading from standard input. UTF-8 characters" << std::endl
+ << "outside ascii range are supported only if input comes from a file." << std::endl;
+ }
+ }
+#endif
if(out_arg != -1)
{
@@ -354,6 +379,19 @@ int main(int argc, char **argv)
return EXIT_FAILURE;
}
}
+#ifdef _MSC_VER
+ else
+ {
+ hfst::set_console_cp_to_utf8();
+ if (!silentFlag)
+ {
+ std::cerr << "hfst-proc: warning: Writing to standard input. UTF-8 characters" << std::endl
+ << "outside ascii range are supported only if output is redirected to a file." << std::endl;
+ }
+ }
+#endif
+
+
CapitalizationMode capitalization_mode;
switch(capitalization)
diff --git a/tools/src/hfst-proc/tokenizer.cc b/tools/src/hfst-proc/tokenizer.cc
index 88685c9..1be38c7 100644
--- a/tools/src/hfst-proc/tokenizer.cc
+++ b/tools/src/hfst-proc/tokenizer.cc
@@ -82,27 +82,28 @@ TokenIOStream::read_utf8_char()
std::string
TokenIOStream::read_utf8_char(std::istream& is)
{
+ std::string retval;
unsigned short u8len = 0;
int c = is.peek();
if(is.eof())
- return "";
+ return retval;
if (c <= 127)
u8len = 1;
else if ( (c & (128 + 64 + 32 + 16)) == (128 + 64 + 32 + 16) )
u8len = 4;
- else if ( (c & (128 + 64 + 32 )) == (128 + 64 + 32) )
+ else if ( (c & (128 + 64 + 32 )) == (128 + 64 + 32) )
u8len = 3;
else if ( (c & (128 + 64 )) == (128 + 64))
u8len = 2;
else
stream_error("Invalid UTF-8 character found");
- char next_u8[u8len+1];
- is.get(next_u8, u8len+1, '\0');
- next_u8[u8len] = '\0';
+ retval.resize(u8len+1);
+ is.get(&retval[0], u8len+1, '\0');
+ retval.resize(strlen(&retval[0]));
- return std::string(next_u8);
+ return retval;
}
bool
diff --git a/tools/src/hfst-proc2.cc b/tools/src/hfst-proc2.cc
index 8a257f4..839ac4c 100644
--- a/tools/src/hfst-proc2.cc
+++ b/tools/src/hfst-proc2.cc
@@ -27,6 +27,7 @@
#include <vector>
#include <map>
#include <string>
+#include <set>
using std::string;
using std::vector;
@@ -48,23 +49,37 @@ using std::pair;
#include "inc/globals-unary.h"
bool blankline_separated = true;
+bool print_all = false;
+bool print_weights = false;
std::string tokenizer_filename;
enum OutputFormat {
- xerox
+ tokenize,
+ xerox,
+ cg,
+ finnpos
};
-OutputFormat output_format = xerox;
+OutputFormat output_format = tokenize;
+
+using hfst_ol::Location;
+using hfst_ol::LocationVector;
+using hfst_ol::LocationVectorVector;
void
print_usage()
{
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
- fprintf(message_out, "Usage: %s [OPTIONS...] TOKENIZER\n"
+ fprintf(message_out, "Usage: %s [--segment | --xerox | --cg] [OPTIONS...] RULESET\n"
"perform matching/lookup on text streams\n"
"\n", program_name);
print_common_program_options(message_out);
fprintf(message_out,
" -n --newline Newline as input separator (default is blank line)\n"
- " -x --xerox Xerox output (default)\n");
+ " -a --print-all Print nonmatching text\n"
+ " -w --print-weight Print weights\n"
+ " --segment Segmenting / tokenization mode (default)\n"
+ " --xerox Xerox output\n"
+ " --cg cg output\n"
+ " --finnpos FinnPos output\n");
fprintf(message_out,
"Use standard streams for input and output (for now).\n"
"\n"
@@ -76,6 +91,122 @@ print_usage()
fprintf(message_out, "\n");
}
+void print_no_output(std::string const & input, std::ostream & outstream)
+{
+ if (output_format == tokenize) {
+ outstream << input;
+ } else if (output_format == xerox) {
+ outstream << input << "\t" << input << "+?";
+ } else if (output_format == cg) {
+ outstream << "\"<>\"" << std::endl << input << "\t\"" << input << "\" ?";
+ }
+// std::cerr << "from print_no_output\n";
+ outstream << "\n\n";
+}
+
+void print_nonmatching_sequence(std::string const & str, std::ostream & outstream)
+{
+ if (output_format == tokenize) {
+ outstream << str;
+ } else if (output_format == xerox) {
+ outstream << str << "\t" << str << "+?";
+ } else if (output_format == cg) {
+ outstream << "\"<>\"" << std::endl << str << "\t\"" << str << "\" ?";
+ } else if (output_format == finnpos) {
+ outstream << str << "\t_\t_\t_\t_";
+ }
+// std::cerr << "from print_nonmatching_sequence\n";
+ outstream << "\n";
+}
+
+void print_location_vector(LocationVector const & locations, std::ostream & outstream)
+{
+ if (output_format == tokenize && locations.size() != 0) {
+ outstream << locations.at(0).input;
+ if (print_weights) {
+ outstream << "\t" << locations.at(0).weight;
+ }
+ outstream << std::endl;
+ } else if (output_format == cg && locations.size() != 0) {
+ // Print the cg cohort header
+ outstream << "\"<" << locations.at(0).input << ">\"" << std::endl;
+ for (LocationVector::const_iterator loc_it = locations.begin();
+ loc_it != locations.end(); ++loc_it) {
+ // For the most common case, eg. analysis strings that begin with the original input,
+ // we try to do what cg tools expect and surround the original input with double quotes.
+ // Otherwise we omit the double quotes and assume the rule writer knows what he's doing.
+ if (loc_it->output.find(loc_it->input) == 0) {
+ // The nice case obtains
+ outstream << "\t\"" << loc_it->input << "\"" <<
+ loc_it->output.substr(loc_it->input.size(), std::string::npos);
+ } else {
+ outstream << "\t" << loc_it->output;
+ }
+ if (print_weights) {
+ outstream << "\t" << loc_it->weight;
+ }
+ outstream << std::endl;
+ }
+ } else if (output_format == xerox) {
+ for (LocationVector::const_iterator loc_it = locations.begin();
+ loc_it != locations.end(); ++loc_it) {
+ outstream << loc_it->input << "\t" << loc_it->output;
+ if (print_weights) {
+ outstream << "\t" << loc_it->weight;
+ }
+ outstream << std::endl;
+ }
+ } else if (output_format == finnpos) {
+ std::set<std::string> tags;
+ std::set<std::string> lemmas;
+ for (LocationVector::const_iterator loc_it = locations.begin();
+ loc_it != locations.end(); ++loc_it) {
+ // Assume the last space is where the tags begin
+ size_t tags_start_at = loc_it->output.find_last_of(" ");
+ if (tags_start_at != std::string::npos) {
+ std::string lemma = loc_it->output.substr(0, tags_start_at);
+ if (lemma.find_first_of(" ") == std::string::npos) {
+ // can't have spaces in lemmas
+ lemmas.insert(lemma);
+ }
+ std::string tag = loc_it->output.substr(tags_start_at + 1);
+ if (tag.find_first_of(" ") == std::string::npos) {
+ // or tags
+ tags.insert(tag);
+ }
+ }
+ }
+ outstream << locations.at(0).input << "\t_\t";
+ // the input and a blank for features
+ if (lemmas.empty()) {
+ outstream << "_";
+ } else {
+ std::string accumulator;
+ for (std::set<std::string>::const_iterator it = lemmas.begin();
+ it != lemmas.end(); ++it) {
+ accumulator.append(*it);
+ accumulator.append(" ");
+ }
+ outstream << accumulator.substr(0, accumulator.size() - 1);
+ }
+ outstream << "\t";
+ if (tags.empty()) {
+ outstream << "_";
+ } else {
+ std::string accumulator;
+ for (std::set<std::string>::const_iterator it = tags.begin();
+ it != tags.end(); ++it) {
+ accumulator.append(*it);
+ accumulator.append(" ");
+ }
+ outstream << accumulator.substr(0, accumulator.size() - 1);
+ }
+ outstream << "\t_";
+ }
+// std::cerr << "from print_location_vector\n";
+ outstream << std::endl;
+}
+
void match_and_print(hfst_ol::PmatchContainer & container,
std::ostream & outstream,
std::string & input_text)
@@ -84,23 +215,26 @@ void match_and_print(hfst_ol::PmatchContainer & container,
// Remove final newline
input_text.erase(input_text.size() -1, 1);
}
- hfst_ol::LocationVectorVector locations = container.locate(input_text);
-
- // Output formatting
- if (output_format == xerox) {
- for(hfst_ol::LocationVectorVector::const_iterator it = locations.begin();
- it != locations.end(); ++it) {
- for (hfst_ol::LocationVector::const_iterator loc_it = it->begin();
- loc_it != it->end(); ++loc_it) {
- outstream << loc_it->input << "\t" << loc_it->output << std::endl;
+ LocationVectorVector locations = container.locate(input_text);
+ if (locations.size() == 0 && print_all) {
+ print_no_output(input_text, outstream);
+ }
+ for(LocationVectorVector::const_iterator it = locations.begin();
+ it != locations.end(); ++it) {
+ if ((it->size() == 1 && it->at(0).output.compare("@_NONMATCHING_@") == 0)) {
+ if (print_all) {
+ print_nonmatching_sequence(it->at(0).input, outstream);
}
- std::cerr << endl;
+ continue;
+ // All nonmatching cases have been handled
}
- } else {
- // More formatting options
+ print_location_vector(*it, outstream);
+ }
+ if (output_format == finnpos) {
+ outstream << std::endl;
}
- outstream << std::endl;
}
+
int process_input(hfst_ol::PmatchContainer & container,
std::ostream & outstream)
@@ -140,11 +274,16 @@ int parse_options(int argc, char** argv)
{
HFST_GETOPT_COMMON_LONG,
{"newline", no_argument, 0, 'n'},
+ {"print-all", no_argument, 0, 'a'},
+ {"print-weights", no_argument, 0, 'w'},
+ {"segment", no_argument, 0, 't'},
{"xerox", no_argument, 0, 'x'},
+ {"cg", no_argument, 0, 'c'},
+ {"finnpos", no_argument, 0, 'f'},
{0,0,0,0}
};
int option_index = 0;
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nxl",
+ char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "nawtxcf",
long_options, &option_index);
if (-1 == c)
{
@@ -158,9 +297,24 @@ int parse_options(int argc, char** argv)
case 'n':
blankline_separated = false;
break;
+ case 'a':
+ print_all = true;
+ break;
+ case 'w':
+ print_weights = true;
+ break;
+ case 't':
+ output_format = tokenize;
+ break;
case 'x':
output_format = xerox;
break;
+ case 'c':
+ output_format = cg;
+ break;
+ case 'f':
+ output_format = finnpos;
+ break;
#include "inc/getopt-cases-error.h"
}
diff --git a/tools/src/hfst-reweight.cc b/tools/src/hfst-reweight.cc
index 4e137a5..a81bf3c 100644
--- a/tools/src/hfst-reweight.cc
+++ b/tools/src/hfst-reweight.cc
@@ -72,6 +72,7 @@ char* input_symbol = 0;
char* output_symbol = 0;
char* symbol = 0;
bool ends_only = false;
+bool arcs_only = false;
char* tsv_file_name = 0;
FILE* tsv_file = 0;
@@ -96,6 +97,7 @@ print_usage()
" -O, --output-symbol=OSYM match arcs with output symbol OSYM\n"
" -S, --symbol=SYM match arcs havins symbol SYM\n"
" -e, --end-states-only match end states only, no arcs\n"
+ " -A, --arcs-only match arcs only, no end states\n"
" -T, --tsv-file=TFILE read reweighting rules from TFILE\n"
"\n");
fprintf(message_out, "\n");
@@ -104,20 +106,25 @@ print_usage()
"elements of addition, multiplication or identity function.\n"
"If LVAL or UVAL are omitted, they default to minimum and maximum "
"values of the weight structure.\n"
- "If ISYM, OSYM or SYM are omitted, they default to value that "
- "matches all arcs.\n"
- "Float values are parsed with strtod(3) and integers strtoul(3)\n"
+ "If ISYM, OSYM or SYM are omitted, they default to a value that "
+ "matches all arcs.\n\n"
+ "Float values are parsed with strtod(3) and integers strtoul(3).\n"
"The functions allowed for FNAME are <cmath> float functions with "
"parameter count of 1 and a matching return value:\n"
- "abs, acos, asin, ... sqrt, tan, tanh\n"
+ "abs, acos, asin, ... sqrt, tan, tanh\n\n"
"The precedence of operands follows the formula "
- "BVAL * FNAME(w) + AVAL\n"
- "The formula is applied iff\n"
+ "BVAL * FNAME(w) + AVAL.\n"
+ "The formula is applied iff:\n"
"((LVAL <= w) && (w <= UVAL)),\n"
"where w is weight of arc, and \n"
"(ISYM == i) && (OSYM == o) && ((SYM == i) || (SYM == o)) ^^ \n"
- "(end state && -e).\n"
- "\n");
+ "(end state && -e).\n\n"
+ "TFILE should contain lines with tab-separated pairs of SYM and "
+ "AVAL or BVAL. AVAL values must be preceded by a + character, "
+ "BVAL should be given as plain digits. "
+ "Comment lines starting with # and empty lines are ignored.\n\n"
+ "Weights are by default modified for all arcs and end states,\n"
+ "unless option --end-states-only or --arcs-only is used.\n");
fprintf(message_out, "\n");
print_report_bugs();
fprintf(message_out, "\n");
@@ -145,14 +152,15 @@ parse_options(int argc, char** argv)
{"input-symbol", required_argument, 0, 'I'},
{"output-symbol", required_argument, 0, 'O'},
{"symbol", required_argument, 0, 'S'},
- {"end-state-only", required_argument, 0, 'e'},
+ {"end-states-only", no_argument, 0, 'e'},
+ {"arcs-only", no_argument, 0, 'A'},
{"tsv", required_argument, 0, 'T'},
{0,0,0,0}
};
int option_index = 0;
// add tool-specific options here
char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT
- HFST_GETOPT_UNARY_SHORT "a:b:F:l:u:I:O:S:eT:",
+ HFST_GETOPT_UNARY_SHORT "a:b:F:l:u:I:O:S:eT:A",
long_options, &option_index);
if (-1 == c)
{
@@ -256,6 +264,9 @@ parse_options(int argc, char** argv)
case 'e':
ends_only = true;
break;
+ case 'A':
+ arcs_only = true;
+ break;
case 'T':
tsv_file_name = hfst_strdup(optarg);
break;
@@ -263,6 +274,12 @@ parse_options(int argc, char** argv)
}
}
+ if (arcs_only && ends_only)
+ {
+ error(EXIT_FAILURE, 0, "Options '--arcs-only' and '--end-states-only' cannot be used at the same time");
+ return EXIT_FAILURE;
+ }
+
#include "inc/check-params-common.h"
#include "inc/check-params-unary.h"
if (funcname == 0)
@@ -292,7 +309,7 @@ reweight(float w, const char* i, const char* o)
}
if ((i == 0) && (o == 0))
{
- if (!ends_only)
+ if (arcs_only)
{
return w;
}
@@ -373,7 +390,7 @@ do_reweight(HfstTransducer& trans)
arc->get_output_symbol().c_str()));
replication.add_transition(rebuilt[source_state], nu);
}
- source_state++;
+ source_state++;
}
trans = HfstTransducer(replication, trans.get_type());
return trans;
@@ -409,7 +426,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
free(symbol);
addition = 0;
multiplier = 1;
- char* line;
+ char* line = NULL;
size_t len = 0;
size_t linen = 0;
verbose_printf("Reading reweights from %s\n", tsv_file_name);
@@ -420,6 +437,10 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
{
continue;
}
+ if (*line == '#')
+ {
+ continue;
+ }
const char* tab = strstr(line, "\t");
if (NULL == tab)
{
@@ -449,8 +470,11 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
multiplier = hfst_strtoweight(weightspec);
}
free(weightspec);
+ verbose_printf("Modifying weights %f < w < %f as %f * %s(w) + %f for symbol %s\n",
+ lower_bound, upper_bound, multiplier, funcname, addition, symbol);
trans = do_reweight(trans);
} // getline
+ free(line);
hfst_set_name(trans, trans, "reweight");
hfst_set_formula(trans, trans, "W");
} // if tsv_file
@@ -496,15 +520,19 @@ int main( int argc, char **argv ) {
}
if (input_symbol)
{
- verbose_printf("only if first symbol is %s\n", input_symbol);
+ verbose_printf("only if input symbol is %s\n", input_symbol);
}
if (output_symbol)
{
- verbose_printf("only if second symbol is %s\n", output_symbol);
+ verbose_printf("only if output symbol is %s\n", output_symbol);
}
if (ends_only)
{
- verbose_printf("only on final weights");
+ verbose_printf("only on final weights, no arcs\n");
+ }
+ if (arcs_only)
+ {
+ verbose_printf("only on arc weights, no end states\n");
}
// here starts the buffer handling part
HfstInputStream* instream = NULL;
diff --git a/tools/src/hfst-split.cc b/tools/src/hfst-split.cc
index 26c0fec..e1d4292 100644
--- a/tools/src/hfst-split.cc
+++ b/tools/src/hfst-split.cc
@@ -121,6 +121,7 @@ parse_options(int argc, char** argv)
free(inputfilename);
inputfilename = hfst_strdup("<stdin>");
}
+ inputNamed = true;
break;
case 'p':
free(prefix);
@@ -160,6 +161,7 @@ process_stream(HfstInputStream& instream)
//outstream->open();
HfstTransducer trans(instream);
*outstream << trans;
+ outstream->flush();
outstream->close();
delete outstream;
free(outfilename);
diff --git a/tools/src/hfst-string-conversions.cc b/tools/src/hfst-string-conversions.cc
index caafc8b..32c75ef 100644
--- a/tools/src/hfst-string-conversions.cc
+++ b/tools/src/hfst-string-conversions.cc
@@ -1,14 +1,131 @@
#ifdef WINDOWS
-#include <windows.h>
-#include <string>
+# include <windows.h>
+# include <string>
+#endif // WINDOWS
+
+#include <cstdarg>
+#include <cstdio>
+
+#include <iostream> // for debugging
+
namespace hfst
{
- std::string wide_string_to_string(const std::wstring & wstr)
+#ifdef WINDOWS
+ /* std::string wide_string_to_string(const std::wstring & wstr)
{
- int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
+ int size_needed = WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(), NULL, 0, NULL, NULL);
std::string str( size_needed, 0 );
- WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &str[0], size_needed, NULL, NULL);
+ WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(), &str[0], size_needed, NULL, NULL);
return str;
- }
-}
-#endif
+ }*/
+#endif // WINDOWS*/
+
+#ifdef WINDOWS
+ bool output_to_console = false;
+ void print_output_to_console(bool val) { output_to_console = val; }
+ bool is_output_printed_to_console() { return output_to_console; }
+#else
+ void print_output_to_console(bool val) { (void)val; }
+ bool is_output_printed_to_console() { return false; }
+#endif // WINDOWS
+
+ int hfst_fprintf(FILE * stream, const char * format, ...)
+ {
+ va_list args;
+ va_start(args, format);
+#ifdef WINDOWS
+ if (output_to_console && (stream == stdout || stream == stderr))
+ {
+ char buffer [1024];
+ int r = vsprintf(buffer, format, args);
+ va_end(args);
+ if (r < 0)
+ return r;
+ HANDLE stdHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+ if (stream == stderr)
+ stdHandle = GetStdHandle(STD_ERROR_HANDLE);
+
+ std::string pstr(buffer);
+ DWORD numWritten = 0;
+ int wchars_num =
+ MultiByteToWideChar(CP_UTF8 , 0 , pstr.c_str() , -1, NULL , 0 );
+ wchar_t* wstr = new wchar_t[wchars_num];
+ MultiByteToWideChar(CP_UTF8 , 0 ,
+ pstr.c_str() , -1, wstr , wchars_num );
+ int retval = WriteConsoleW(stdHandle, wstr, wchars_num-1, &numWritten, NULL);
+ delete[] wstr;
+
+ return retval;
+ }
+ else
+ {
+ int retval = vfprintf(stream, format, args);
+ va_end(args);
+ return retval;
+ }
+#else
+ int retval = vfprintf(stream, format, args);
+ va_end(args);
+ return retval;
+#endif // WINDOWS
+ }
+
+#ifdef WINDOWS
+ bool get_line_from_console(std::string & str, size_t buffer_size, bool keep_newline /* = false*/)
+ {
+ bool DEBUG = false;
+ SetConsoleCP(65001);
+ const HANDLE stdIn = GetStdHandle(STD_INPUT_HANDLE);
+ WCHAR * buffer = new WCHAR [buffer_size];
+ DWORD numRead = 0;
+ if (ReadConsoleW(stdIn, buffer, size_t(buffer_size/4), &numRead, NULL))
+ {
+ if (DEBUG) { std::cerr << "get_line_from_console: numRead is " << numRead << std::endl; }
+
+ int size_needed = WideCharToMultiByte(CP_UTF8, 0, buffer, (int)numRead, NULL, 0, NULL, NULL);
+ if (DEBUG) { std::cerr << "conversions: size_needed is " << size_needed << std::endl; }
+ CHAR * strbuf = new CHAR [size_needed];
+ WideCharToMultiByte(CP_UTF8, 0, buffer, (int)numRead, strbuf, size_needed, NULL, NULL);
+ delete buffer;
+ strbuf[size_needed] = 0; // NULL-terminate the buffer
+ str = std::string(strbuf);
+
+ if (DEBUG) { std::cerr << "get_line_from_console: size of str is now (1) " << str.size() << std::endl; }
+
+ if (str[0] == (char)26 || str[0] == (char)4) // control+Z, control+D
+ return false;
+
+ // Get rid of carriage returns and newlines.
+
+ if (str.size() == 0)
+ return true;
+
+ // Get rid of carriage return
+ if (str.size() > 1)
+ {
+ if (str[str.size()-2] == '\r')
+ str.erase(str.size()-2, 1);
+ }
+
+ if (str[str.size()-1] != '\n')
+ return true;
+
+ if (keep_newline)
+ return true;
+
+ str.erase(str.size()-1);
+ return true;
+
+ if (DEBUG) { std::cerr << "get_line_from_console: size of str is now (2) " << str.size() << std::endl; }
+ return true;
+ }
+ return false;
+ }
+
+ void set_console_cp_to_utf8()
+ {
+ SetConsoleCP(65001);
+ }
+#endif // WINDOWS
+
+} // namespace hfst
diff --git a/tools/src/hfst-string-conversions.h b/tools/src/hfst-string-conversions.h
index 7fce470..87b1866 100644
--- a/tools/src/hfst-string-conversions.h
+++ b/tools/src/hfst-string-conversions.h
@@ -1,7 +1,33 @@
#ifdef WINDOWS
-#include <string>
+# include <string>
+#endif // WINDOWS
+
+#include <cstdio>
+
namespace hfst
{
- std::string wide_string_to_string(const std::wstring & wstr);
+
+#ifdef WINDOWS
+ /* Convert utf-8 string \a wstr into an ordinary string. */
+ //std::string wide_string_to_string(const std::wstring & wstr);
+ /* Get a line from console input and store it into \a str.
+ \a buffer_size defines maximum of input length.
+ Return whether the read operation was succesful. */
+ bool get_line_from_console(std::string & str, size_t buffer_size, bool keep_newline = false);
+
+ void set_console_cp_to_utf8();
+#endif // WINDOWS
+
+ /* Whether hfst_fprintf prints directly to console output.
+ Has no effect on linux or mac. */
+ void print_output_to_console(bool val);
+ /* Return whether hfst_fprintf prints directly to console output.
+ The default is false. */
+ bool is_output_printed_to_console();
+
+ /* Wrapper around fprintf that prints to console output,
+ is_output_printed_to_console is true.
+ On linux and mac, calls always fprintf directly. */
+ int hfst_fprintf(FILE * stream, const char * format, ...);
+
}
-#endif
diff --git a/tools/src/hfst-strings2fst.cc b/tools/src/hfst-strings2fst.cc
index 0969ac0..860b7f0 100644
--- a/tools/src/hfst-strings2fst.cc
+++ b/tools/src/hfst-strings2fst.cc
@@ -38,7 +38,13 @@ using std::pair;
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
+
#include <math.h>
#include <errno.h>
@@ -263,7 +269,7 @@ process_stream(HfstOutputStream& outstream)
HfstBasicTransducer disjunction;
size_t line_n = 0;
- HfstStrings2FstTokenizer
+ hfst::HfstStrings2FstTokenizer
multichar_symbol_tokenizer(multichar_symbols,std::string(epsilonname));
while (hfst_getline(&line, &len, inputfile) != -1)
@@ -312,7 +318,7 @@ process_stream(HfstOutputStream& outstream)
{ spv = multichar_symbol_tokenizer.tokenize_string_pair
(line,has_spaces); }
}
- catch (const UnescapedColsFound &e)
+ catch (const hfst::UnescapedColsFound &e)
{
if (pairstrings)
{
@@ -421,7 +427,7 @@ int main( int argc, char **argv )
multichar_symbol_filename);
std::ifstream multichar_in(multichar_symbol_filename);
(void)multichar_in.peek();
- if (not multichar_in.good())
+ if (! multichar_in.good())
{ error(EXIT_FAILURE, errno,"Multichar symbol file can't be read."); }
char multichar_line[1000];
while (multichar_in.good())
diff --git a/tools/src/hfst-subtract.cc b/tools/src/hfst-subtract.cc
index 607e08a..0d9a0dc 100644
--- a/tools/src/hfst-subtract.cc
+++ b/tools/src/hfst-subtract.cc
@@ -236,6 +236,7 @@ subtract_streams(HfstInputStream& firststream, HfstInputStream& secondstream,
firststream.close();
secondstream.close();
+ outstream.flush();
outstream.close();
return EXIT_SUCCESS;
}
diff --git a/tools/src/hfst-summarize.cc b/tools/src/hfst-summarize.cc
index ec4b1a5..73edaf4 100644
--- a/tools/src/hfst-summarize.cc
+++ b/tools/src/hfst-summarize.cc
@@ -69,7 +69,7 @@ print_usage()
print_common_unary_program_options(message_out);
// fprintf(message_out, (tool-specific options and short descriptions)
fprintf(message_out, "Summarize options:\n");
- fprintf(message_out, " -p, --print-symbol-pair-statistics=N Print info about symbol pairs that occur\n");
+ fprintf(message_out, " -S, --print-symbol-pair-statistics=N Print info about symbol pairs that occur\n");
fprintf(message_out, " at most N times (default is infinity)\n");
fprintf(message_out, "\n");
print_common_unary_program_parameter_instructions(message_out);
@@ -555,6 +555,9 @@ process_stream(HfstInputStream& instream)
}
fprintf(outfile, "\n");
}
+
+ } // if verbose
+
// ADDED
if (print_symbol_pair_statistics)
{
@@ -575,9 +578,7 @@ process_stream(HfstInputStream& instream)
}
fprintf(outfile, "\n");
}
-
delete trans;
- }
}
fprintf(outfile, "\nRead " SIZE_T_SPECIFIER " transducers in total.\n", transducer_n);
diff --git a/tools/src/hfst-tail.cc b/tools/src/hfst-tail.cc
index 856bd79..f4ed4ed 100644
--- a/tools/src/hfst-tail.cc
+++ b/tools/src/hfst-tail.cc
@@ -177,6 +177,7 @@ process_stream(HfstInputStream& instream, HfstOutputStream& outstream)
}
}
}
+ outstream.flush();
instream.close();
outstream.close();
return EXIT_SUCCESS;
diff --git a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc b/tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc
index f0e7482..1506312 100644
--- a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc
+++ b/tools/src/hfst-twolc/src/alphabet_src/Alphabet.cc
@@ -1,4 +1,4 @@
-//! @file Alphabet.cc
+ //! @file Alphabet.cc
//!
//! @author Miikka Silfverberg
//!
@@ -18,6 +18,10 @@
#include "Alphabet.h"
+#ifdef HAVE_XFSM
+ #define Alphabet TwolCAlphabet
+#endif
+
void Alphabet::define_set(const std::string &name,const SymbolRange &elements)
{ sets[name] = elements; }
@@ -34,11 +38,11 @@ const OtherSymbolTransducer &Alphabet::get_transducer(const SymbolPair &pair)
bool Alphabet::is_pair(const std::string &input,const std::string &output)
{
- if (input == "__HFST_TWOLC_?" and output == "__HFST_TWOLC_?")
+ if (input == "__HFST_TWOLC_?" && output == "__HFST_TWOLC_?")
{ return true; }
- if (diacritics.has_element(input) and input == output)
+ if (diacritics.has_element(input) && input == output)
{ return true; }
- if (diacritics.has_element(input) and output == "__HFST_TWOLC_?")
+ if (diacritics.has_element(input) && output == "__HFST_TWOLC_?")
{ return true; }
if (input == "__HFST_TWOLC_?")
{ return output_symbols.has_element(output); }
@@ -60,15 +64,15 @@ void Alphabet::define_singleton_set(const std::string &name)
bool Alphabet::is_set_pair(const SymbolPair &pair) const
{
return
- pair.first.find("__HFST_TWOLC_SET_NAME=") != std::string::npos or
+ pair.first.find("__HFST_TWOLC_SET_NAME=") != std::string::npos ||
pair.second.find("__HFST_TWOLC_SET_NAME=") != std::string::npos;
}
const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
{
- if (not sets.has_key(pair.first))
+ if (! sets.has_key(pair.first))
{ define_singleton_set(pair.first); }
- if (not sets.has_key(pair.second))
+ if (! sets.has_key(pair.second))
{ define_singleton_set(pair.second); }
const std::string &input = pair.first;
@@ -80,26 +84,26 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
{
pair_transducer.apply(&HfstTransducer::disjunct,
OtherSymbolTransducer(input,input));
- if (input != output and output != TWOLC_EPSILON and
+ if (input != output && output != TWOLC_EPSILON &&
output != TWOLC_UNKNOWN)
{ std::cerr << "Warning: Diacritic " << input << " in pair "
<< input << ":" << output << " will correspond 0."
<< std::endl; }
}
- else if (input == TWOLC_UNKNOWN and output == TWOLC_UNKNOWN)
+ else if (input == TWOLC_UNKNOWN && output == TWOLC_UNKNOWN)
{
for (HandySet<SymbolPair>::const_iterator it = alphabet_set.begin();
it != alphabet_set.end();
++it)
- {
- if (is_set_pair(*it))
- { continue; }
+ {
+ if (is_set_pair(*it))
+ { continue; }
- pair_transducer.apply(&HfstTransducer::disjunct,
- OtherSymbolTransducer(it->first,it->second));
- }
+ pair_transducer.apply(&HfstTransducer::disjunct,
+ OtherSymbolTransducer(it->first,it->second));
+ }
pair_transducer.apply(&HfstTransducer::disjunct,
- OtherSymbolTransducer(TWOLC_UNKNOWN));
+ OtherSymbolTransducer(TWOLC_UNKNOWN));
}
else if (input == TWOLC_UNKNOWN)
{
@@ -113,14 +117,14 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
jt != alphabet_set.end();
++jt)
{
- if (is_set_pair(*jt))
- { continue; }
+ if (is_set_pair(*jt))
+ { continue; }
if (*it == jt->second)
- {
- pair_transducer.apply(&HfstTransducer::disjunct,
- OtherSymbolTransducer
- (jt->first,jt->second)); }
+ {
+ pair_transducer.apply(&HfstTransducer::disjunct,
+ OtherSymbolTransducer
+ (jt->first,jt->second)); }
}
}
}
@@ -136,14 +140,14 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
jt != alphabet_set.end();
++jt)
{
- if (is_set_pair(*jt))
- { continue; }
-
+ if (is_set_pair(*jt))
+ { continue; }
+
if (*it == jt->first)
- {
- pair_transducer.apply(&HfstTransducer::disjunct,
- OtherSymbolTransducer
- (jt->first,jt->second)); }
+ {
+ pair_transducer.apply(&HfstTransducer::disjunct,
+ OtherSymbolTransducer
+ (jt->first,jt->second)); }
}
}
}
@@ -161,10 +165,10 @@ const OtherSymbolTransducer &Alphabet::compute(const SymbolPair &pair)
++jt)
{
if (is_pair(*it,*jt))
- {
- pair_transducer.apply(&HfstTransducer::disjunct,
- OtherSymbolTransducer(*it,*jt));
- }
+ {
+ pair_transducer.apply(&HfstTransducer::disjunct,
+ OtherSymbolTransducer(*it,*jt));
+ }
}
}
}
diff --git a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.h b/tools/src/hfst-twolc/src/alphabet_src/Alphabet.h
index 4fc0fb3..5530afb 100644
--- a/tools/src/hfst-twolc/src/alphabet_src/Alphabet.h
+++ b/tools/src/hfst-twolc/src/alphabet_src/Alphabet.h
@@ -28,6 +28,10 @@
#include "../HfstTwolcDefs.h"
#include "../rule_src/OtherSymbolTransducer.h"
+#ifdef HAVE_XFSM
+ #define Alphabet TwolCAlphabet
+#endif
+
class Alphabet
{
protected:
diff --git a/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc b/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
index cbcfccc..0c4b25d 100644
--- a/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
+++ b/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
@@ -210,7 +210,7 @@ int CommandLine::parse_options(int argc, char** argv)
}
- if (not inputNamed)
+ if (! inputNamed)
{
if ((argc - optind) == 1)
{
@@ -288,7 +288,7 @@ std::istream &CommandLine::set_input_file(void)
if (has_input_file)
{
input_file = new std::ifstream(input_file_name.c_str());
- if (not input_file->good())
+ if (! input_file->good())
{
std::cerr << "File " << input_file_name << " could not be opened!"
<< std::endl;
@@ -305,7 +305,7 @@ std::ostream &CommandLine::set_output_file(void)
if (has_output_file)
{
output_file = new std::ofstream(output_file_name.c_str());
- if (not output_file->good())
+ if (! output_file->good())
{
std::cerr << "File " << output_file_name << " could not be opened!"
<< std::endl;
diff --git a/tools/src/hfst-twolc/src/commandline_src/CommandLine.h b/tools/src/hfst-twolc/src/commandline_src/CommandLine.h
index edb7bc2..61f993b 100644
--- a/tools/src/hfst-twolc/src/commandline_src/CommandLine.h
+++ b/tools/src/hfst-twolc/src/commandline_src/CommandLine.h
@@ -17,10 +17,17 @@
#ifdef HAVE_CONFIG_H
# include <config.h>
+#else
+# define PACKAGE_STRING ""
#endif
#include <iostream>
-#include <getopt.h>
+
+#ifndef _MSC_VER
+# include <getopt.h>
+#else
+# include "../../../hfst-getopt.h"
+#endif
#include "HfstTransducer.h"
using hfst::ImplementationType;
diff --git a/tools/src/hfst-twolc/src/hfst-twolc.bat b/tools/src/hfst-twolc/src/hfst-twolc.bat
index 4db7af1..cf29fa1 100644
--- a/tools/src/hfst-twolc/src/hfst-twolc.bat
+++ b/tools/src/hfst-twolc/src/hfst-twolc.bat
@@ -1,2 +1,2 @@
@echo off
-HFST_INSTALLATION_DIRECTORY\htwolcpre1.exe %* | HFST_INSTALLATION_DIRECTORY\htwolcpre2.exe %* | HFST_INSTALLATION_DIRECTORY\htwolcpre3.exe %*
+htwolcpre1.exe %* | htwolcpre2.exe %* | htwolcpre3.exe %*
diff --git a/tools/src/hfst-twolc/src/htwolcpre1.yy b/tools/src/hfst-twolc/src/htwolcpre1.yy
index c597dda..94725e9 100644
--- a/tools/src/hfst-twolc/src/htwolcpre1.yy
+++ b/tools/src/hfst-twolc/src/htwolcpre1.yy
@@ -549,7 +549,7 @@ void reduce_symbol_pair(bool no_definitions)
// encountered.
void increase_line_counter(void)
{
- while (not symbol_queue.empty() &&
+ while (! symbol_queue.empty() &&
symbol_queue.front() == "__HFST_TWOLC_\\n")
{
++line_number;
@@ -600,9 +600,9 @@ void reduce_queue(bool variable_symbol)
// string representations of rules.
// 3. We push back variable symbols into rule_symbol_vector, which stores
// rule-variable names and values.
- if (not variable_symbol)
+ if (! variable_symbol)
{
- if (not rules_start)
+ if (! rules_start)
{
std::cout << get_symbol_queue_front() << " ";
pop_symbol_queue();
@@ -629,7 +629,7 @@ int main(int argc, char * argv[])
CommandLine command_line(argc,argv);
- if (command_line.help or command_line.version)
+ if (command_line.help || command_line.version)
{
if (command_line.version)
{ command_line.print_version(); }
@@ -642,14 +642,14 @@ int main(int argc, char * argv[])
command_line.print_usage();
exit(0);
}
- if (not command_line.be_quiet)
+ if (! command_line.be_quiet)
{
- if (not command_line.has_input_file)
+ if (! command_line.has_input_file)
{ std::cerr << "Reading input from STDIN." << std::endl; }
else
{ std::cerr << "Reading input from " << command_line.input_file_name
<< "." << std::endl; }
- if (not command_line.has_output_file)
+ if (! command_line.has_output_file)
{ std::cerr << "Writing output to STDOUT." << std::endl; }
else
{ std::cerr << "Writing output to " << command_line.output_file_name
diff --git a/tools/src/hfst-twolc/src/htwolcpre2.yy b/tools/src/hfst-twolc/src/htwolcpre2.yy
index 6e798b9..11201ac 100644
--- a/tools/src/hfst-twolc/src/htwolcpre2.yy
+++ b/tools/src/hfst-twolc/src/htwolcpre2.yy
@@ -237,20 +237,20 @@ void insert_alphabet_pairs(const HandyDeque<std::string> &symbol_queue,
++it)
{
//If we found a symbol pair, we insert it into symbol_pair_set.
- if ((*it == "__HFST_TWOLC_0" or
- *it == "__HFST_TWOLC_.#." or
- *it == "__HFST_TWOLC_#" or
- *it == "__HFST_TWOLC_SPACE" or
- *it == "__HFST_TWOLC_TAB" or
+ if ((*it == "__HFST_TWOLC_0" ||
+ *it == "__HFST_TWOLC_.#." ||
+ *it == "__HFST_TWOLC_#" ||
+ *it == "__HFST_TWOLC_SPACE" ||
+ *it == "__HFST_TWOLC_TAB" ||
it->find("__HFST_TWOLC_") == std::string::npos)
- and
+ &&
*(it+1) == "__HFST_TWOLC_:"
- and
- (*(it+2) == "__HFST_TWOLC_0" or
- *(it+2) == "__HFST_TWOLC_.#." or
- *(it+2) == "__HFST_TWOLC_#" or
- *(it+2) == "__HFST_TWOLC_SPACE" or
- *(it+2) == "__HFST_TWOLC_TAB" or
+ &&
+ (*(it+2) == "__HFST_TWOLC_0" ||
+ *(it+2) == "__HFST_TWOLC_.#." ||
+ *(it+2) == "__HFST_TWOLC_#" ||
+ *(it+2) == "__HFST_TWOLC_SPACE" ||
+ *(it+2) == "__HFST_TWOLC_TAB" ||
(it+2)->find("__HFST_TWOLC_") == std::string::npos))
{
std::string input_symbol = *it == "__HFST_TWOLC_#" ? "#" : *it;
@@ -304,7 +304,7 @@ int main(int argc, char * argv[])
#endif
CommandLine command_line(argc,argv);
- if (command_line.help or command_line.usage or command_line.version)
+ if (command_line.help || command_line.usage || command_line.version)
{ exit(0); }
//yydebug = 1;
input_reader.set_input(std::cin);
diff --git a/tools/src/hfst-twolc/src/htwolcpre3.yy b/tools/src/hfst-twolc/src/htwolcpre3.yy
index 47315ca..1b0d437 100644
--- a/tools/src/hfst-twolc/src/htwolcpre3.yy
+++ b/tools/src/hfst-twolc/src/htwolcpre3.yy
@@ -58,6 +58,10 @@
size_t line_number = 1;
InputReader input_reader(line_number);
+#ifdef HAVE_XFSM
+ #define Alphabet TwolCAlphabet
+#endif
+
// The Alphabet of the grammar
Alphabet alphabet;
@@ -395,7 +399,7 @@ ALPHABET_PAIR_LIST: /* empty */
PAIR: PAIR_SYMBOL COLON PAIR_SYMBOL
{
- if (std::string("__HFST_TWOLC_0") == $1 and
+ if (std::string("__HFST_TWOLC_0") == $1 &&
std::string("__HFST_TWOLC_0") == $3)
{ $$ = new OtherSymbolTransducer(HFST_EPSILON); }
else if (std::string("__HFST_TWOLC_#") == $1)
@@ -524,7 +528,7 @@ unsigned int get_second_number(const std::string &s)
void message(const std::string &m)
{
- if (not silent)
+ if (! silent)
{ std::cerr << m << std::endl; }
}
@@ -553,7 +557,7 @@ int main(int argc, char * argv[])
try
{
CommandLine command_line(argc,argv);
- if (command_line.help or command_line.usage or command_line.version)
+ if (command_line.help || command_line.usage || command_line.version)
{ exit(0); }
if (command_line.has_debug_file)
{ input_reader.set_input(command_line.set_input_file()); }
@@ -574,7 +578,7 @@ int main(int argc, char * argv[])
{ exit(exit_code); }
message("Compiling and storing rules.");
- if (not command_line.has_output_file)
+ if (! command_line.has_output_file)
{
HfstOutputStream stdout_(command_line.format);
grammar->compile_and_store(stdout_);
diff --git a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc b/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc
index 54f2529..5e78fef 100644
--- a/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc
+++ b/tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc
@@ -59,7 +59,7 @@ OtherSymbolTransducer wbize(const OtherSymbolTransducer &t)
bool ConflictResolvingLeftArrowRule::conflicts_this
(const ConflictResolvingLeftArrowRule &another,StringVector &v)
{
- return not context.is_empty_intersection(wbize(another.context),v);
+ return ! context.is_empty_intersection(wbize(another.context),v);
}
bool ConflictResolvingLeftArrowRule::resolvable_conflict
diff --git a/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc b/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc
index 6809539..fe37864 100644
--- a/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc
+++ b/tools/src/hfst-twolc/src/rule_src/OtherSymbolTransducer.cc
@@ -191,7 +191,7 @@ OtherSymbolTransducer &OtherSymbolTransducer::harmonize_diacritics
it != diacritics.end();
++it)
{
- if (t_alphabet.find(*it) != t_alphabet.end() and
+ if (t_alphabet.find(*it) != t_alphabet.end() &&
alphabet.find(*it) == alphabet.end())
{ missing_diacritics.insert(*it); }
}
@@ -277,31 +277,31 @@ void OtherSymbolTransducer::check_pair(const std::string &input_symbol,
if (input_symbol == TWOLC_IDENTITY)
{ is_broken = false; }
// other:other is a valid pair.
- else if (input_symbol == HFST_UNKNOWN and output_symbol == HFST_UNKNOWN)
+ else if (input_symbol == HFST_UNKNOWN && output_symbol == HFST_UNKNOWN)
{ is_broken = false; }
// eps:eps is valid
- else if (input_symbol == TWOLC_EPSILON and output_symbol == TWOLC_EPSILON)
+ else if (input_symbol == TWOLC_EPSILON && output_symbol == TWOLC_EPSILON)
{ is_broken = false; }
// 0:0 is a valid pair.
- else if (input_symbol == HFST_EPSILON and output_symbol == HFST_EPSILON)
+ else if (input_symbol == HFST_EPSILON && output_symbol == HFST_EPSILON)
{ is_broken = false; }
// diamond:diamond is a valid pair.
else if (input_symbol == TWOLC_DIAMOND)
{ is_broken = false; }
// other:X is valid, iff X is an output symbol or 0.
else if (input_symbol == HFST_UNKNOWN)
- { is_broken = not (output_symbol == TWOLC_EPSILON or
+ { is_broken = ! (output_symbol == TWOLC_EPSILON ||
output_symbols.has_element(output_symbol)); }
// X:other is valid, iff X is an input symbol or 0.
else if (output_symbol == HFST_UNKNOWN)
- { is_broken = not (input_symbol == TWOLC_EPSILON or
+ { is_broken = ! (input_symbol == TWOLC_EPSILON ||
input_symbols.has_element(input_symbol)); }
// 0:X is valid, iff X is an output symbol.
else if (input_symbol == TWOLC_EPSILON)
- { is_broken = not output_symbols.has_element(output_symbol); }
+ { is_broken = ! output_symbols.has_element(output_symbol); }
// X:0 is valid, iff X is an input symbol or a diacritic.
else if (output_symbol == TWOLC_EPSILON)
- { is_broken = not input_symbols.has_element(input_symbol); }
+ { is_broken = ! input_symbols.has_element(input_symbol); }
// X:X is valid if X is a diacritic.
else if (diacritics.has_element(input_symbol) /*and
input_symbol == output_symbol*/)
@@ -309,7 +309,7 @@ void OtherSymbolTransducer::check_pair(const std::string &input_symbol,
// X:Y is valid iff it has been declared in the alphabet.
else
{ is_broken =
- not symbol_pairs.has_element(SymbolPair(input_symbol,output_symbol)); }
+ ! symbol_pairs.has_element(SymbolPair(input_symbol,output_symbol)); }
if (is_broken)
{ std::cerr << "Unknown pair: "
<< input_symbol << " " << output_symbol << std::endl; }
@@ -347,7 +347,7 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
if (another.is_broken)
{ throw UndefinedSymbolPairsFound(); }
OtherSymbolTransducer another_copy(another);
- if (not diacritics.empty())
+ if (! diacritics.empty())
{
harmonize_diacritics(another_copy);
another_copy.harmonize_diacritics(*this);
@@ -367,7 +367,7 @@ OtherSymbolTransducer &OtherSymbolTransducer::apply
if (another.is_broken)
{ throw UndefinedSymbolPairsFound(); }
OtherSymbolTransducer another_copy(another);
- if (not diacritics.empty())
+ if (! diacritics.empty())
{
harmonize_diacritics(another_copy);
another_copy.harmonize_diacritics(*this);
@@ -556,7 +556,7 @@ OtherSymbolTransducer OtherSymbolTransducer::get_inverse_of_upper_projection
symbol_pairs.begin();
kt != symbol_pairs.end(); ++kt)
{
- if (kt->first == input and has_symbol(fst,kt->second))
+ if (kt->first == input && has_symbol(fst,kt->second))
{ add_transition(new_fst,state,target,input,kt->second); }
}
if (input == TWOLC_EPSILON)
@@ -653,7 +653,7 @@ bool have_common_string(HfstState state1,HfstState state2,
HandySet<StatePair> &visited_pairs,
StringVector &v)
{
- if (fst1.is_final_state(state1) and fst2.is_final_state(state2))
+ if (fst1.is_final_state(state1) && fst2.is_final_state(state2))
{ return true; }
const HfstBasicTransducer::HfstTransitions &fst1_transitions = fst1[state1];
@@ -679,7 +679,7 @@ bool have_common_string(HfstState state1,HfstState state2,
{
StatePair state_pair(fst1_transition_map[symbol_pair],
it->get_target_state());
- if (not visited_pairs.has_element(state_pair))
+ if (! visited_pairs.has_element(state_pair))
{
v.push_back(symbol_pair.first + ":" + symbol_pair.second);
visited_pairs.insert(state_pair);
@@ -701,7 +701,7 @@ bool OtherSymbolTransducer::is_empty_intersection
HfstBasicTransducer another_fst(another.transducer);
HandySet<StatePair> visited_pairs;
visited_pairs.insert(StatePair(0,0));
- return not have_common_string(0,0,this_fst,another_fst,visited_pairs,v);
+ return ! have_common_string(0,0,this_fst,another_fst,visited_pairs,v);
}
bool OtherSymbolTransducer::is_subset(const OtherSymbolTransducer &another)
diff --git a/tools/src/hfst-twolc/src/rule_src/Rule.cc b/tools/src/hfst-twolc/src/rule_src/Rule.cc
index 8673a8f..26bde0a 100644
--- a/tools/src/hfst-twolc/src/rule_src/Rule.cc
+++ b/tools/src/hfst-twolc/src/rule_src/Rule.cc
@@ -42,7 +42,7 @@ Rule::Rule(const std::string &name,
it != v.end();
++it)
{
- if (not (*it)->empty())
+ if (! (*it)->empty())
{
rule_transducer.apply
(&HfstTransducer::intersect,(*it)->rule_transducer);
diff --git a/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc b/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc
index 7956fb3..6d94a84 100644
--- a/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc
+++ b/tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc
@@ -26,7 +26,7 @@ TwolCGrammar::TwolCGrammar(bool be_quiet,
be_quiet(be_quiet),
be_verbose(be_verbose)
{
- left_arrow_rule_container.set_report_left_arrow_conflicts(not be_quiet);
+ left_arrow_rule_container.set_report_left_arrow_conflicts(! be_quiet);
left_arrow_rule_container.set_resolve_left_arrow_conflicts
(resolve_left_conflicts);
right_arrow_rule_container.set_report_right_arrow_conflicts
@@ -175,12 +175,12 @@ void TwolCGrammar::add_rule(const std::string &name,
void TwolCGrammar::compile_and_store(HfstOutputStream &out)
{
- if (not be_quiet)
+ if (! be_quiet)
{ std::cerr << "Compiling rules." << std::endl; }
- left_arrow_rule_container.compile(std::cerr,(not be_quiet) and be_verbose);
- right_arrow_rule_container.compile(std::cerr,(not be_quiet) and be_verbose);
- other_rule_container.compile(std::cerr,(not be_quiet) and be_verbose);
+ left_arrow_rule_container.compile(std::cerr,(! be_quiet) && be_verbose);
+ right_arrow_rule_container.compile(std::cerr,(! be_quiet) && be_verbose);
+ other_rule_container.compile(std::cerr,(! be_quiet) && be_verbose);
for (StringRuleSetMap::const_iterator it = name_to_rule_subcases.begin();
it != name_to_rule_subcases.end();
@@ -190,9 +190,9 @@ void TwolCGrammar::compile_and_store(HfstOutputStream &out)
it->second.end()))); }
compiled_rule_container.add_missing_symbols_freely(diacritics);
- if (not be_quiet)
+ if (! be_quiet)
{ std::cerr << "Storing rules." << std::endl; }
- compiled_rule_container.store(out,std::cerr,(not be_quiet) and be_verbose);
+ compiled_rule_container.store(out,std::cerr,(! be_quiet) && be_verbose);
}
#ifdef TEST_TWOL_C_GRAMMAR
@@ -232,6 +232,10 @@ ImplementationType transducer_type
symbols.insert(SymbolPair("b","c"));*/
//g.set_alphabet(symbols);
+#ifdef HAVE_XFSM
+ #define Alphabet TwolCAlphabet
+#endif
+
Alphabet alphabet;
alphabet.define_alphabet_pair(SymbolPair("a","b"));
alphabet.define_alphabet_pair(SymbolPair("a","d"));
diff --git a/tools/src/hfst-twolc/src/scanner1.ll b/tools/src/hfst-twolc/src/scanner1.ll
index 7bcb0e4..674c9c2 100644
--- a/tools/src/hfst-twolc/src/scanner1.ll
+++ b/tools/src/hfst-twolc/src/scanner1.ll
@@ -176,14 +176,14 @@ in/{RESERVED_SYMBOL} { return IN; }
return SYMBOL;
}
[?]/[:] {
- if (not regexp_start)
+ if (! regexp_start)
{ return QUESTION_MARK; }
// Any symbol.
symbol_queue.push_back("__HFST_TWOLC_?");
return SYMBOL;
}
[?]/{RESERVED_EXC_COL} {
- if (not regexp_start)
+ if (! regexp_start)
{ return QUESTION_MARK; }
// Any symbol.
symbol_queue.push_back("__HFST_TWOLC_?");
@@ -285,7 +285,7 @@ in/{RESERVED_SYMBOL} { return IN; }
}
\( {
// Beginning of an optional bracketed regex.
- if (not where_seen)
+ if (! where_seen)
{
symbol_queue.push_back("__HFST_TWOLC_(");
reduce_queue();
@@ -294,7 +294,7 @@ in/{RESERVED_SYMBOL} { return IN; }
}
\) {
// End of an optional bracketed regex.
- if (not where_seen)
+ if (! where_seen)
{
symbol_queue.push_back("__HFST_TWOLC_)");
reduce_queue();
diff --git a/tools/src/hfst-twolc/src/string_src/string_manipulation.cc b/tools/src/hfst-twolc/src/string_src/string_manipulation.cc
index c02fe6c..b629818 100644
--- a/tools/src/hfst-twolc/src/string_src/string_manipulation.cc
+++ b/tools/src/hfst-twolc/src/string_src/string_manipulation.cc
@@ -77,7 +77,7 @@ std::string unescape_and_remove_white_space(std::string str)
std::string unquote(const std::string &str)
{
- if (str.size() < 2 or str[0] != '"' or str[str.size()-1] != '"')
+ if (str.size() < 2 || str[0] != '"' || str[str.size()-1] != '"')
{ throw FaultyStringInput("unquote",str); }
// Return the substring of str spanning from the
// second to the next to final character.
diff --git a/tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h b/tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h
index e73dd7a..116770e 100644
--- a/tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h
+++ b/tools/src/hfst-twolc/src/variable_src/ConstContainerIterator.h
@@ -97,7 +97,7 @@ template <class T> class ConstContainerIterator
//! @brief Test inequality with @another.
bool operator!=(const ConstContainerIterator &another) const
- { return not (*this == another); };
+ { return ! (*this == another); };
//! @brief Increment.
virtual int operator++(void)
@@ -114,7 +114,7 @@ template <class T> class ConstContainerIterator
break;
}
}
- if (not found_a_non_final_iterator)
+ if (! found_a_non_final_iterator)
{ iterator_vector = end_iterator_vector; }
return 1;
};
diff --git a/tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h b/tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h
index 170c1b0..4926eef 100644
--- a/tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h
+++ b/tools/src/hfst-twolc/src/variable_src/MixedConstContainerIterator.h
@@ -63,14 +63,14 @@ public ConstContainerIterator<T>
MixedConstContainerIterator(const ConstContainerIterator<T> &another)
{
ConstContainerIterator<T>::operator=(another);
- while (didnt_end() and equal_indices())
+ while (didnt_end() && equal_indices())
{ operator++(); }
}
//! @brief Increment.
int operator++(void)
{
do { ConstContainerIterator<T>::operator++(); }
- while (didnt_end() and equal_indices());
+ while (didnt_end() && equal_indices());
return 1;
}
MixedConstContainerIterator operator+(size_t i) const
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc b/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc
index 79abe1d..a9ef6b2 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc
+++ b/tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc
@@ -31,7 +31,7 @@ std::string RuleSymbolVector::replace_variables(void)
std::string symbol = *it;
if (symbol.find("__HFST_TWOLC_RULE_NAME") != std::string::npos)
{
- if (not vvm.empty())
+ if (! vvm.empty())
{
symbol.insert
(symbol.size()-1,
@@ -61,7 +61,7 @@ std::string RuleSymbolVector::replace_variables(const RuleCenter ¢er)
std::string symbol = *it;
if (symbol.find("__HFST_TWOLC_RULE_NAME") != std::string::npos)
{
- if (not vvm.empty())
+ if (! vvm.empty())
{
symbol.insert
(symbol.size()-1,
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleVariables.cc b/tools/src/hfst-twolc/src/variable_src/RuleVariables.cc
index 3af523a..b0131f8 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleVariables.cc
+++ b/tools/src/hfst-twolc/src/variable_src/RuleVariables.cc
@@ -73,7 +73,7 @@ void RuleVariables::clear(void)
bool RuleVariables::empty(void) const
{
return
- freely_blocks.begin() == freely_blocks.end() and
- matched_blocks.begin() == matched_blocks.end() and
+ freely_blocks.begin() == freely_blocks.end() &&
+ matched_blocks.begin() == matched_blocks.end() &&
mixed_blocks.begin() == mixed_blocks.end();
}
diff --git a/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc b/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc
index 58fffd4..9dace53 100644
--- a/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc
+++ b/tools/src/hfst-twolc/src/variable_src/RuleVariablesConstIterator.cc
@@ -72,7 +72,7 @@ bool RuleVariablesConstIterator::operator==
bool RuleVariablesConstIterator::operator!=
(const RuleVariablesConstIterator &another)
-{ return not (*this == another); }
+{ return ! (*this == another); }
void RuleVariablesConstIterator::operator++(void)
{
diff --git a/tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h b/tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h
index a21d860..5675786 100644
--- a/tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h
+++ b/tools/src/hfst-twolc/src/variable_src/VariableValueIterator.h
@@ -55,7 +55,7 @@ template <class IT> class VariableValueIterator
//! @brief Assign @a another.
VariableValueIterator &operator=(const VariableValueIterator<IT> &another)
{
- if (not (this == &another))
+ if (! (this == &another))
{
variable = another.variable;
it = another.it;
@@ -65,11 +65,11 @@ template <class IT> class VariableValueIterator
//! @brief Test equality with another.
bool operator==(const VariableValueIterator<IT> &another) const
- { return variable == another.variable and it == another.it; }
+ { return variable == another.variable && it == another.it; }
//! @brief Test unequality with another.
bool operator!=(const VariableValueIterator<IT> &another) const
- { return not (*this == another); }
+ { return ! (*this == another); }
//! Increment.
void operator++(void)
diff --git a/tools/src/hfst-txt2fst.cc b/tools/src/hfst-txt2fst.cc
index 35c0b84..2d99541 100644
--- a/tools/src/hfst-txt2fst.cc
+++ b/tools/src/hfst-txt2fst.cc
@@ -32,7 +32,12 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <getopt.h>
+
+#ifdef _MSC_VER
+# include "hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
#include "HfstTransducer.h"
#include "HfstInputStream.h"
@@ -49,7 +54,7 @@ using hfst::implementations::HfstBasicTransducer;
#include "inc/globals-unary.h"
// add tools-specific variables here
static hfst::ImplementationType output_format = hfst::UNSPECIFIED_TYPE;
-static bool read_prolog=false;
+static bool read_prolog_format=false;
// whether numbers are used instead of symbol names
static bool use_numbers=false; // not used
// printname for epsilon
@@ -127,7 +132,7 @@ break;
use_numbers = true;
break;
case 'p':
- read_prolog = true;
+ read_prolog_format = true;
break;
case 'f':
output_format = hfst_parse_format_name(optarg);
@@ -172,8 +177,25 @@ process_stream(HfstOutputStream& outstream)
{
verbose_printf("Reading transducer table " SIZE_T_SPECIFIER "...\n", transducer_n);
}
- if (read_prolog)
+ if (read_prolog_format)
{
+
+ if (output_format == hfst::XFSM_TYPE)
+ {
+ try {
+ HfstTransducer * t = HfstTransducer::prolog_file_to_xfsm_transducer(inputfilename);
+ outstream << *t;
+ delete(t);
+ outstream.flush();
+ break;
+ }
+ catch (const HfstException & e) {
+ error(EXIT_FAILURE, 0, "Error when trying to convert text file (prolog) into xfsm format\n"
+ "Note that the file may contain only one transducer in text format\n");
+ return EXIT_FAILURE;
+ }
+ }
+
try {
HfstBasicTransducer fsm =
HfstBasicTransducer::read_in_prolog_format(inputfile, linecount);
@@ -243,6 +265,9 @@ int main( int argc, char **argv )
case hfst::FOMA_TYPE:
verbose_printf("Using foma as output handler\n");
break;
+ case hfst::XFSM_TYPE:
+ verbose_printf("Using xfsm as output handler\n");
+ break;
case hfst::HFST_OL_TYPE:
verbose_printf("Using optimized lookup output\n");
break;
@@ -253,6 +278,28 @@ int main( int argc, char **argv )
error(EXIT_FAILURE, 0, "Unknown format cannot be used as output\n");
return EXIT_FAILURE;
}
+
+ if (output_format == hfst::XFSM_TYPE)
+ {
+ if (strcmp(outfilename, "<stdout>") == 0) {
+ error(EXIT_FAILURE, 0, "Writing to standard output not supported for xfsm transducers,\n"
+ "use 'hfst-txt2fst [--output|-o] OUTFILE' instead");
+ return EXIT_FAILURE;
+ }
+ if (!read_prolog_format) {
+ error(EXIT_FAILURE, 0, "Writing in att format not supported for xfsm transducers,\n"
+ "use '--prolog' instead");
+ return EXIT_FAILURE;
+ }
+ if (strcmp(inputfilename, "<stdin>") == 0) {
+ error(EXIT_FAILURE, 0, "Reading prolog format from standard input not supported for xfsm transducers,\n"
+ "use 'hfst-txt2fst [--input|-i] INFILE' instead");
+ return EXIT_FAILURE;
+ }
+ else {}
+ }
+
+
// here starts the buffer handling part
HfstOutputStream* outstream = (outfile != stdout) ?
new HfstOutputStream(outfilename, output_format) :
diff --git a/tools/src/parsers/Makefile.am b/tools/src/parsers/Makefile.am
index 48666eb..f265dd1 100644
--- a/tools/src/parsers/Makefile.am
+++ b/tools/src/parsers/Makefile.am
@@ -46,7 +46,7 @@ endif
noinst_HEADERS = XfstCompiler.h xfst-utils.h xfst-parser.hh
-HFST_COMMON_SRC=../hfst-program-options.cc ../hfst-commandline.cc ../hfst-tool-metadata.cc ../HfstStrings2FstTokenizer.cc ../hfst-file-to-mem.cc ../hfst-string-conversions.cc
+HFST_COMMON_SRC=../hfst-program-options.cc ../hfst-commandline.cc ../hfst-tool-metadata.cc ../HfstStrings2FstTokenizer.cc ../hfst-file-to-mem.cc ../hfst-string-conversions.cc xfst_help_message.cc
bin_PROGRAMS=$(MAYBE_XFST)
@@ -62,5 +62,5 @@ xfst-parser.cc: xfst-parser.yy
xfst-lexer.cc: xfst-lexer.ll
-EXTRA_DIST=init_help.cc cmd.h abbrcmd.h help_message.cc xfst-parser.cc xfst-lexer.cc xfst-parser.hh
+EXTRA_DIST=init_help.cc cmd.h abbrcmd.h xfst_help_message.h xfst-parser.cc xfst-lexer.cc xfst-parser.hh
diff --git a/tools/src/parsers/XfstCompiler.cc b/tools/src/parsers/XfstCompiler.cc
index 31344e3..4587df0 100644
--- a/tools/src/parsers/XfstCompiler.cc
+++ b/tools/src/parsers/XfstCompiler.cc
@@ -33,13 +33,14 @@ using std::stack;
#include <cstdarg>
-#ifndef WINDOWS
+#ifndef _WIN32
#include <glob.h>
#else
#include <windows.h>
- #include "../hfst-string-conversions.h"
#endif // WINDOWS
+#include "../hfst-string-conversions.h"
+
#include "XfstCompiler.h"
#include "xfst-utils.h"
#include "xfst-parser.hh"
@@ -56,7 +57,7 @@ extern FILE* hxfstin;
extern int hxfstparse(void);
extern int hxfstlex(void);
-class yy_buffer_state;
+struct yy_buffer_state;
typedef yy_buffer_state *YY_BUFFER_STATE;
extern YY_BUFFER_STATE hxfst_scan_string(const char*);
extern void hxfst_delete_buffer(YY_BUFFER_STATE);
@@ -65,6 +66,7 @@ extern void hxfst_delete_buffer(YY_BUFFER_STATE);
using hfst::implementations::HfstBasicTransducer;
using hfst::implementations::HfstBasicTransition;
+using hfst::hfst_fprintf;
#define GET_TOP(x) HfstTransducer * x = this->top(); if ((x) == NULL) { xfst_lesser_fail(); return *this; }
#define PROMPT_AND_RETURN_THIS prompt(); return *this;
@@ -78,7 +80,7 @@ using hfst::implementations::HfstBasicTransition;
#define LOOKUP_CYCLE_CUTOFF "5"
#define PRINT_WORDS_CYCLE_CUTOFF "5"
-#include "help_message.cc"
+#include "xfst_help_message.h"
namespace hfst {
namespace xfst {
@@ -240,47 +242,6 @@ namespace xfst {
return hfst_fprintf(stream, formatter.c_str(), weight);
}
- int XfstCompiler::hfst_fprintf(FILE * stream, const char * format, ...) const
- {
- va_list args;
- va_start(args, format);
-#ifdef WINDOWS
- if (output_to_console_ && (stream == stdout || stream == stderr))
- {
- char buffer [1024];
- int r = vsprintf(buffer, format, args);
- va_end(args);
- if (r < 0)
- return r;
- HANDLE stdHandle = GetStdHandle(STD_OUTPUT_HANDLE);
- if (stream == stderr)
- stdHandle = GetStdHandle(STD_ERROR_HANDLE);
-
- std::string pstr(buffer);
- DWORD numWritten = 0;
- int wchars_num =
- MultiByteToWideChar(CP_UTF8 , 0 , pstr.c_str() , -1, NULL , 0 );
- wchar_t* wstr = new wchar_t[wchars_num];
- MultiByteToWideChar(CP_UTF8 , 0 ,
- pstr.c_str() , -1, wstr , wchars_num );
- int retval = WriteConsoleW(stdHandle, wstr, wchars_num-1, &numWritten, NULL);
- delete[] wstr;
-
- return retval;
- }
- else
- {
- int retval = vfprintf(stream, format, args);
- va_end(args);
- return retval;
- }
-#else
- int retval = vfprintf(stream, format, args);
- va_end(args);
- return retval;
-#endif
- }
-
void XfstCompiler::xfst_fail()
{
if (variables_["quit-on-fail"] == "ON")
@@ -371,12 +332,93 @@ namespace xfst {
return retval;
}
- XfstCompiler&
+ bool is_valid_string(const StringVector & sv)
+ {
+ // map features to latest values
+ std::map<std::string, std::string> values;
+ // and keep track of features whose values have been negatively set
+ std::set<std::string> negative_values;
+
+ for (StringVector::const_iterator it = sv.begin(); it != sv.end(); it++)
+ {
+ if (FdOperation::is_diacritic(*it))
+ {
+ std::string opstr = FdOperation::get_operator(*it);
+ assert(opstr.size() == 1);
+ char op = opstr[0];
+ std::string feat = FdOperation::get_feature(*it);
+ std::string val = FdOperation::get_value(*it);
+
+ bool is_negatively_set = (negative_values.find(feat) != negative_values.end());
+
+ switch(op) {
+ case 'P': // positive set
+ values[feat] = val;
+ break;
+ case 'N': // negative set
+ values[feat] = val;
+ negative_values.insert(feat);
+ break;
+ case 'R': // require
+ if (val == "") // empty require
+ if (values[feat] == "")
+ { return false; }
+ else { // nonempty require
+ if (is_negatively_set || (values[feat] != val)) { return false; }
+ }
+ break;
+ case 'D': // disallow
+ if (val == "") { // empty disallow
+ if (values[feat] != "") {
+ return false;
+ }
+ }
+ else {
+ if ((!is_negatively_set) && (values[feat] == val)) { // nonempty disallow
+ return false;
+ }
+ }
+ break;
+ case 'C': // clear
+ values[feat] = "";
+ break;
+ case 'U': // unification
+ if(values[feat] == "" || /* if the feature is unset or */
+ ((!is_negatively_set) && (values[feat] == val)) || /* the feature is at
+ this value already
+ or */
+ (is_negatively_set &&
+ (values[feat] != val)) /* the feature is
+ negatively set
+ to something
+ else */
+ )
+ {
+ values[feat] = val;
+ }
+ else
+ {
+ return false;
+ }
+ break;
+ default:
+ std::cerr << "ERROR: line: " << __LINE__ << std::endl;
+ throw; // for the compiler's peace of mind
+ break;
+ }
+ }
+ }
+ return true;
+ }
+
+ bool
XfstCompiler::print_paths
(const hfst::HfstOneLevelPaths &paths,
FILE* outfile /* =stdout */,
int n /* = -1*/)
{
+ bool retval = false; // if anything was printed
+
// go through at most n paths
for (hfst::HfstOneLevelPaths::const_iterator it = paths.begin();
n != 0 && it != paths.end(); it++)
@@ -384,6 +426,11 @@ namespace xfst {
hfst::StringVector path = it->second;
bool something_printed = false; // to control printing spaces
+ if ((variables_["obey-flags"] == "ON") && !is_valid_string(path))
+ continue;
+
+ retval = true; // something will be printed
+
// go through the path
for (hfst::StringVector::const_iterator p = path.begin();
p != path.end(); p++)
@@ -418,15 +465,17 @@ namespace xfst {
} // at most n paths gone through
- return *this;
+ return retval;
}
- XfstCompiler&
+ bool
XfstCompiler::print_paths
(const hfst::HfstTwoLevelPaths &paths,
FILE* outfile /* =stdout */,
int n /* = -1*/)
{
+ bool retval = false; // if anything was printed
+
// go through at most n paths
for (hfst::HfstTwoLevelPaths::const_iterator it = paths.begin();
n != 0 && it != paths.end(); it++)
@@ -434,6 +483,15 @@ namespace xfst {
hfst::StringPairVector path = it->second;
bool something_printed = false; // to control printing spaces
+ if (variables_["obey-flags"] == "ON")
+ {
+ StringVector path_input = hfst::symbols::to_string_vector(path, true /*input side*/);
+ if (!is_valid_string(path_input))
+ continue;
+ }
+
+ retval = true; // something will be printed
+
// go through the path
for (hfst::StringPairVector::const_iterator p = path.begin();
p != path.end(); p++)
@@ -477,7 +535,7 @@ namespace xfst {
} // at most n paths gone through
- return *this;
+ return retval;
}
static float string_to_float(const std::string & str)
@@ -534,10 +592,12 @@ namespace xfst {
HfstOneLevelPaths paths = extract_output_paths(results);
- this->print_paths(paths);
- if (paths.empty()) {
- hfst_fprintf(outstream_, "???\n");
- }
+ bool printed = this->print_paths(paths);
+ if (!printed)
+ {
+ hfst_fprintf(outstream_, "???\n");
+ }
+
return *this;
}
@@ -555,10 +615,11 @@ namespace xfst {
paths = t->lookup(std::string(token), cutoff);
}
- this->print_paths(*paths);
- if (paths->empty()) {
- hfst_fprintf(outstream_, "???\n");
- }
+ bool printed = this->print_paths(*paths);
+ if (!printed)
+ {
+ hfst_fprintf(outstream_, "???\n");
+ }
delete paths;
return *this;
@@ -2312,13 +2373,13 @@ namespace xfst {
stack_.pop();
std::string liststr(list);
- if (liststr == "NOTHING")
+ if (liststr == "\"NOTHING\"") // list is given in quoted format: "foo" "bar" ...
liststr = "";
- // use regex parser: `[ [TR] , s , L ]
+ // use regex parser: `[ [TR] , "s" , L ]
xre_.define("TempXfstTransducerName", *top);
std::string subst_regex("`[ [TempXfstTransducerName] , ");
- subst_regex += std::string(target) + " , " + liststr + " ]";
+ subst_regex += "\"" + std::string(target) + "\" , " + liststr + " ]";
HfstTransducer * substituted = xre_.compile(subst_regex);
xre_.undefine("TempXfstTransducerName");
@@ -2395,7 +2456,7 @@ namespace xfst {
XfstCompiler&
XfstCompiler::print_dir(const char* globdata, FILE* outfile)
{
-#ifndef WINDOWS
+#ifndef _WIN32
glob_t globbuf;
int rv = glob(globdata, 0, NULL, &globbuf);
@@ -2921,6 +2982,7 @@ namespace xfst {
{
GET_TOP(tmp);
HfstBasicTransducer basic(*tmp);
+#ifdef WINDOWS
if (output_to_console_ && (outfile == stdout))
{
std::ostringstream ostr;
@@ -2928,6 +2990,7 @@ namespace xfst {
hfst_fprintf(outfile, ostr.str().c_str());
}
else
+#endif
{
basic.write_in_xfst_format(outfile, variables_["print-weight"] == "ON");
}
@@ -3593,6 +3656,14 @@ namespace xfst {
while (!stack_.empty())
{
HfstTransducer* t = stack_.top();
+
+ if (t->get_type() != result->get_type())
+ {
+ hfst_fprintf(stderr, "Stack contains transducers whose type differs.\n");
+ xfst_lesser_fail();
+ break;
+ }
+
switch (operation)
{
case INTERSECT_NET:
@@ -3972,7 +4043,7 @@ namespace xfst {
if (use_readline_ && file == stdin)
{
char *buf = NULL; // result from readline
- rl_bind_key('\t',rl_abort); // disable auto-complet
+ rl_bind_key('\t',rl_insert); // disable auto-complet
if((buf = readline(promptstr.c_str())) != NULL)
{
@@ -3983,12 +4054,23 @@ namespace xfst {
}
#endif
- hfst_fprintf(stderr, "%s", promptstr.c_str());
+ hfst_fprintf(outstream_, "%s", promptstr.c_str());
#ifdef WINDOWS
// if we are reading directly from console
if ((file == stdin) && read_interactive_text_from_stdin_)
{
+ std::string str("");
+ size_t bufsize = 1000;
+ if (hfst::get_line_from_console(str, bufsize))
+ {
+ return strdup(str.c_str());
+ }
+ else
+ {
+ return NULL;
+ }
+ /*
SetConsoleCP(65001);
const HANDLE stdIn = GetStdHandle(STD_INPUT_HANDLE);
WCHAR buffer[0x1000];
@@ -3997,12 +4079,16 @@ namespace xfst {
{
std::wstring wstr(buffer);
std::string linestr = wide_string_to_string(wstr);
+ if (linestr[0] == (char)26) // control+Z
+ {
+ return NULL;
+ }
return strdup(linestr.c_str());
}
else
{
return NULL;
- }
+ }*/
}
#endif
@@ -4617,6 +4703,7 @@ namespace xfst {
XfstCompiler::setOutputToConsole(bool value)
{
output_to_console_ = value;
+ hfst::print_output_to_console(output_to_console_);
return *this;
}
bool
@@ -4654,7 +4741,12 @@ namespace xfst {
{
if (verbose_prompt_ && verbose_)
{
+ // On windows, prompt is always printed to console. On other platforms,
+ // this has no effect.
+ bool val = hfst::is_output_printed_to_console();
+ hfst::print_output_to_console(true);
hfst_fprintf(outstream_, "hfst[" SIZE_T_SPECIFIER "]: ", stack_.size());
+ hfst::print_output_to_console(val);
}
return *this;
}
diff --git a/tools/src/parsers/XfstCompiler.h b/tools/src/parsers/XfstCompiler.h
index e73e2b8..fc8c53b 100644
--- a/tools/src/parsers/XfstCompiler.h
+++ b/tools/src/parsers/XfstCompiler.h
@@ -511,11 +511,11 @@ class XfstCompiler
const char* get_print_symbol(const char* symbol);
//! @brief Print \a n first paths (or all, if n is negative)
//! from \a paths to \a outfile.
- XfstCompiler& print_paths(const hfst::HfstTwoLevelPaths &paths,
+ bool print_paths(const hfst::HfstTwoLevelPaths &paths,
FILE* outfile=stdout, int n=-1);
//! @brief Print \a n first paths (or all, if n is negative)
//! from \a paths to \a outfile.
- XfstCompiler& print_paths(const hfst::HfstOneLevelPaths &paths,
+ bool print_paths(const hfst::HfstOneLevelPaths &paths,
FILE* outfile=stdout, int n=-1);
// A method used by function print_longest_string_or_its_size.
XfstCompiler& print_one_string_or_its_size
@@ -626,9 +626,6 @@ class XfstCompiler
//! @brief Print weight.
int hfst_print_weight(FILE * stream, float weight);
- //! @brief Wrapper for fprintf.
- int hfst_fprintf(FILE * stream, const char * format, ...) const;
-
//! @brief Get current readline history index.
int current_history_index();
diff --git a/tools/src/parsers/hfst-xfst.cc b/tools/src/parsers/hfst-xfst.cc
index 3b1af24..48f9ffd 100644
--- a/tools/src/parsers/hfst-xfst.cc
+++ b/tools/src/parsers/hfst-xfst.cc
@@ -32,7 +32,12 @@
#include <readline/history.h>
#endif
-#include <getopt.h>
+#ifdef _MSC_VER
+# include "../hfst-getopt.h"
+#else
+# include <getopt.h>
+#endif
+
#include "hfst-commandline.h"
#include "hfst-program-options.h"
#include "hfst-tool-metadata.h"
@@ -43,8 +48,9 @@ static hfst::ImplementationType output_format = hfst::UNSPECIFIED_TYPE;
static char* scriptfilename = NULL;
static char* startupfilename = NULL;
static std::vector<char*> execute_commands;
-static bool pipemode = false;
-static bool output_to_console = false;
+static bool pipe_input = false;
+static bool pipe_output = false; // this has no effect on non-windows platforms
+
#ifdef HAVE_READLINE
static bool use_readline = true;
#else
@@ -69,16 +75,28 @@ print_usage()
" -f, --format=FMT Write result using FMT as backend format\n"
" -F, --scriptfile=FILE Read commands from FILE, and quit\n"
" -l, --startupfile=FILE Read commands from FILE on startup\n"
- " -p, --pipe-mode Read commands from standard input (non-interactive)\n"
+ " -p, --pipe-mode[=STREAM] Control input and output streams\n"
" -r, --no-readline Do not use readline library for input\n"
" -w, --print-weight Print weights for each operation\n"
- " -k, --output-to-console Output directly to console (Windows-specific)\n"
+ // " -k, --no-console Do not output directly to console (Windows-specific)\n"
"\n"
"Option --execute can be invoked many times.\n"
"If FMT is not given, OpenFst's tropical format will be used.\n"
"The possible values for FMT are { foma, openfst-tropical, openfst-log, sfst }.\n"
"Readline library, if enabled when configuring, is used for input by default.\n"
- "Input files are always treated as UTF-8.\n");
+ "Input files are always treated as UTF-8.\n"
+ "\n"
+ "STREAM can be { input, output, both }. If not given, defaults to {both}.\n"
+#ifdef _MSC_VER
+ "If input file is not specified with -F, input is read interactively via the\n"
+ "console, i.e. line by line from the user. If you redirect input from a file,\n"
+ "use --pipe-mode=input. Output is by default printed to the console. If you\n"
+ "redirect output to a file, use --pipe-mode=output.\n");
+#else
+ "If input file is not specified with -F, input is read interactively line by\n"
+ "line from the user. If you redirect input from a file, use --pipe-mode=input.\n"
+ "--pipe-mode=output is ignored on non-windows platforms.\n");
+#endif
fprintf(message_out, "\n");
print_report_bugs();
fprintf(message_out, "\n");
@@ -100,15 +118,15 @@ parse_options(int argc, char** argv)
{"scriptfile", required_argument, 0, 'F'},
{"execute", required_argument, 0, 'e'},
{"startupfile", required_argument, 0, 'l'},
- {"pipe-mode", no_argument, 0, 'p'},
+ {"pipe-mode", optional_argument, 0, 'p'},
{"no-readline", no_argument, 0, 'r'},
{"print-weight", no_argument, 0, 'w'},
- {"output-to-console", no_argument, 0, 'k'},
+ // {"no-console", no_argument, 0, 'k'},
{0,0,0,0}
};
int option_index = 0;
// add tool-specific options here
- char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "f:F:e:l:prwk",
+ char c = getopt_long(argc, argv, HFST_GETOPT_COMMON_SHORT "f:F:e:l:p::rwk",
long_options, &option_index);
if (-1 == c)
{
@@ -144,16 +162,34 @@ parse_options(int argc, char** argv)
case 'F':
scriptfilename = hfst_strdup(optarg);
break;
+ // todo: on windows getopt_long does not support unicode:
+ // e.g. option -e 'regex U;' where U is a unicode character is not possible
case 'e':
execute_commands.push_back(hfst_strdup(optarg));
break;
case 'l':
startupfilename = hfst_strdup(optarg);
break;
+
case 'p':
- pipemode = true;
- use_readline = false;
+ if (optarg == NULL)
+ { pipe_input = true; pipe_output = true; }
+ else if (strcmp(optarg, "both") == 0 || strcmp(optarg, "BOTH") == 0)
+ { pipe_input = true; pipe_output = true; }
+ else if (strcmp(optarg, "input") == 0 || strcmp(optarg, "INPUT") == 0 ||
+ strcmp(optarg, "in") == 0 || strcmp(optarg, "IN") == 0)
+ { pipe_input = true; }
+ else if (strcmp(optarg, "output") == 0 || strcmp(optarg, "OUTPUT") == 0 ||
+ strcmp(optarg, "out") == 0 || strcmp(optarg, "OUT") == 0)
+ { pipe_output = true; }
+ else
+ { error(EXIT_FAILURE, 0, "--pipe-mode argument %s unrecognised", optarg); }
break;
+
+ // case 'p':
+ //pipe_input = true;
+ //use_readline = false;
+ //break;
case 'r':
use_readline = false;
break;
@@ -161,7 +197,7 @@ parse_options(int argc, char** argv)
print_weight = true;
break;
case 'k':
- output_to_console = true;
+ pipe_output = true;
break;
#include "inc/getopt-cases-error.h"
}
@@ -208,15 +244,22 @@ void insert_zeroes(char * array, unsigned int number)
static bool expression_continues(std::string & expr)
{
size_t s = expr.size();
- if (s > 0 && expr.at(s-1) == '\\')
+
+ // get rid of extra newlines...
+ if (s > 0 && expr.at(s-1) == '\n')
{
- expr.at(s-1) = '\n';
- return true;
+ expr.erase(s-1);
+ }
+ s = expr.size();
+ // and carriage returns
+ if (s > 0 && expr.at(s-1) == '\r')
+ {
+ expr.erase(s-1);
}
- if (s > 1 && expr.at(s-1) == '\r' && expr.at(s-2) == '\\')
+ s = expr.size();
+ if (s > 0 && expr.at(s-1) == '\\')
{
expr.at(s-1) = '\n';
- expr.at(s-2) = '\r';
return true;
}
return false;
@@ -312,7 +355,7 @@ int main(int argc, char** argv)
return EXIT_FAILURE;
}
- if (pipemode && (scriptfilename != NULL))
+ if (pipe_input && (scriptfilename != NULL))
{
error(EXIT_FAILURE, 0 , "--pipe-mode and --scriptfile cannot be used simultaneously\n");
return EXIT_FAILURE;
@@ -341,7 +384,7 @@ int main(int argc, char** argv)
comp.setPromptVerbosity(true);
}
- if (output_to_console)
+ if (!pipe_output)
comp.setOutputToConsole(true);
// If needed, execute scripts given in command line
@@ -365,7 +408,7 @@ int main(int argc, char** argv)
}
}
- if (pipemode)
+ if (pipe_input)
{
verbose_printf("Reading from standard input...\n");
comp.setReadInteractiveTextFromStdin(false);
@@ -396,19 +439,24 @@ int main(int argc, char** argv)
std::string expression = "";
//unsigned int lines = 0;
+ const unsigned int MAX_LINE_LENGTH = 1024;
#ifdef WINDOWS
- SetConsoleCP(65001);
+ /*SetConsoleCP(65001);
const HANDLE stdIn = GetStdHandle(STD_INPUT_HANDLE);
WCHAR buffer[0x1000];
DWORD numRead = 0;
- while (ReadConsoleW(stdIn, buffer, sizeof buffer, &numRead, NULL))
+ while (ReadConsoleW(stdIn, buffer, size_t((0x1000)/4), &numRead, NULL))
{
std::wstring wstr(buffer, numRead);
std::string linestr = hfst::wide_string_to_string(wstr);
- expression += linestr;
+ expression += linestr;*/
+ std::string str("");
+ while(hfst::get_line_from_console(str, MAX_LINE_LENGTH))
+ {
+ expression += str;
+ str = std::string("");
#else
- const unsigned int MAX_LINE_LENGTH = 1024;
char line [MAX_LINE_LENGTH];
insert_zeroes(line, MAX_LINE_LENGTH);
while (cin.getline(line, MAX_LINE_LENGTH))
diff --git a/tools/src/parsers/test/Makefile.am b/tools/src/parsers/test/Makefile.am
index 3fbbab3..72f236a 100644
--- a/tools/src/parsers/test/Makefile.am
+++ b/tools/src/parsers/test/Makefile.am
@@ -29,6 +29,8 @@ EXTRA_DIST=test.sh \
substitute_symbol_4.xfst substitute_symbol_4.att \
substitute_symbol_5.xfst substitute_symbol_5.att \
substitute_symbol_6.xfst substitute_symbol_6.att \
+ substitute_symbol_7.xfst substitute_symbol_7.att \
+ substitute_symbol_8.xfst substitute_symbol_8.att \
substitute_label_1.xfst substitute_label_1.att \
substitute_label_2.xfst substitute_label_2.att \
substitute_label_3.xfst substitute_label_3.att \
diff --git a/tools/src/parsers/test/substitute_symbol_7.att b/tools/src/parsers/test/substitute_symbol_7.att
new file mode 100644
index 0000000..d821b08
--- /dev/null
+++ b/tools/src/parsers/test/substitute_symbol_7.att
@@ -0,0 +1,8 @@
+0 1 bar foo
+1 2 foo foo
+2 3 foo bar
+3
+--
+0 1 foo foo
+0 1 bar bar
+1
diff --git a/tools/src/parsers/test/substitute_symbol_7.xfst b/tools/src/parsers/test/substitute_symbol_7.xfst
new file mode 100644
index 0000000..946a62a
--- /dev/null
+++ b/tools/src/parsers/test/substitute_symbol_7.xfst
@@ -0,0 +1,7 @@
+# flag to normal symbol
+regex bar:"@U.FOO.ON@" "@U.FOO.ON@" "@U.FOO.ON@":bar ;
+substitute symbol foo for @U.FOO.ON@
+write att
+echo --
+sigma net
+write att
diff --git a/tools/src/parsers/test/substitute_symbol_8.att b/tools/src/parsers/test/substitute_symbol_8.att
new file mode 100644
index 0000000..9ee5575
--- /dev/null
+++ b/tools/src/parsers/test/substitute_symbol_8.att
@@ -0,0 +1,7 @@
+0 1 @U.FOO.ON@ @U.FOO.ON@
+1 2 @U.FOO.ON@ @U.FOO.ON@
+2 3 @U.FOO.ON@ @U.FOO.ON@
+3
+--
+0 1 @U.FOO.ON@ @U.FOO.ON@
+1
diff --git a/tools/src/parsers/test/substitute_symbol_8.xfst b/tools/src/parsers/test/substitute_symbol_8.xfst
new file mode 100644
index 0000000..91f329e
--- /dev/null
+++ b/tools/src/parsers/test/substitute_symbol_8.xfst
@@ -0,0 +1,7 @@
+# normal symbol to flag
+regex bar:"@U.FOO.ON@" "@U.FOO.ON@" "@U.FOO.ON@":bar ;
+substitute symbol @U.FOO.ON@ for bar
+write att
+echo --
+sigma net
+write att
diff --git a/tools/src/parsers/test/test.sh b/tools/src/parsers/test/test.sh
index 9f82ab7..09dffc3 100755
--- a/tools/src/parsers/test/test.sh
+++ b/tools/src/parsers/test/test.sh
@@ -1,18 +1,29 @@
#!/bin/sh
-XFST_TOOL="../hfst-xfst -s --pipe-mode"
+# hfst tools needed
+XFST_TOOL="../hfst-xfst -s --pipe-mode=input"
STRINGS2FST="../../hfst-strings2fst -S"
TXT2FST="../../hfst-txt2fst"
COMPARE="../../hfst-compare --quiet"
+HFST_FORMAT="../../hfst-format" # optional
+
+# other tools needed
DIFF="diff --ignore-blank-lines"
-LIST_FORMATS="../../hfst-format --list-formats"
-EXTRA_FILES="tmp startup"
REMOVE="rm -f"
+GREP="grep"
+CAT="cat"
+LS="ls"
+TEST="test"
+TR="tr"
+# ECHO="echo" # present on all platforms?
+
+EXTRA_FILES="tmp startup" # files generated by tests
+# edit this is hfst-format is not available
for format in sfst openfst-tropical foma;
do
- # Test if implementation type is available.
- if ! (${LIST_FORMATS} | grep $format > /dev/null); then
+ # comment this if hfst-format is not available
+ if ! (${HFST_FORMAT} --list-formats | ${GREP} $format > /dev/null); then
continue;
fi
@@ -56,32 +67,32 @@ do
done
## Test that using special symbols in replace rules yields an error message
- if ! (echo 'regex a -> "@_foo_@";' | ../hfst-xfst --pipe-mode -f $format > /dev/null 2> tmp && grep "warning:" tmp > /dev/null); then
+ if ! (echo 'regex a -> "@_foo_@";' | ../hfst-xfst --pipe-mode=input -f $format > /dev/null 2> tmp && ${GREP} "warning:" tmp > /dev/null); then
echo "fail #5";
exit 1;
fi
# silent mode
- if (echo 'regex a -> "@_foo_@";' | ../hfst-xfst --pipe-mode -s -f $format > /dev/null 2> tmp && grep "warning:" tmp > /dev/null); then
+ if (echo 'regex a -> "@_foo_@";' | ../hfst-xfst --pipe-mode=input -s -f $format > /dev/null 2> tmp && ${GREP} "warning:" tmp > /dev/null); then
echo "fail #6";
exit 1;
fi
## Test that using incompatible replace types in parallel rules yields an error message
- if (!(echo 'regex a -> b ,, c (->) d ;' | ../hfst-xfst --pipe-mode -f $format > /dev/null 2> tmp) && grep "failed" tmp > /dev/null); then
+ if (!(echo 'regex a -> b ,, c (->) d ;' | ../hfst-xfst --pipe-mode=input -f $format > /dev/null 2> tmp) && ${GREP} "failed" tmp > /dev/null); then
echo "fail #6.5";
exit 1;
fi
## Test that the transducer info is correct
- if ! (echo 'regex [a|b|c|d|e] ([d|e|f|g]);' | ../hfst-xfst --pipe-mode -f $format > tmp 2> /dev/null); then
+ if ! (echo 'regex [a|b|c|d|e] ([d|e|f|g]);' | ../hfst-xfst --pipe-mode=input -f $format > tmp 2> /dev/null); then
echo "fail #7";
exit 1;
fi
- if (! grep "3 states" tmp > /dev/null); then
+ if (! ${GREP} "3 states" tmp > /dev/null); then
echo "here 1"
exit 1;
fi
- if (! grep "9 arcs" tmp > /dev/null); then
+ if (! ${GREP} "9 arcs" tmp > /dev/null); then
echo "here 2"
exit 1;
fi
@@ -100,19 +111,19 @@ do
substitute_defined_4 substitute_defined_5 substitute_defined_6 \
at_re_1 at_re_2 at_re_3 at_txt at_stxt at_txt_and_stxt at_pl \
quoted_literals replace_identity one_transition_regex merge
- # substitute_symbol_6 fails on sfst
+ # substitute_symbol_6 fails on sfst, substitute_symbol_7 substitute_symbol_8 should be added
# angle_brackets omitted, since xfst and foma handle them differently
do
- rm -f result result1 result2
- if ! (ls $testfile.xfst 2> /dev/null); then
+ ${REMOVE} result result1 result2
+ if ! (${LS} $testfile.xfst 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
- if ! (cat $testfile.xfst | ../hfst-xfst --pipe-mode -q -f $format > result 2> /dev/null); then
+ if ! (${CAT} $testfile.xfst | ../hfst-xfst --pipe-mode=input -q -f $format > result 2> /dev/null); then
echo "ERROR: in compiling "$testfile".xfst"
exit 1;
fi
- if ! (cat result | ${TXT2FST} > tmp1; cat $testfile.att | ${TXT2FST} > tmp2; ); then
+ if ! (${CAT} result | ${TXT2FST} > tmp1; ${CAT} $testfile.att | ${TXT2FST} > tmp2; ); then
echo "ERROR: in compiling "$testfile".att"
exit 1;
fi
@@ -126,16 +137,16 @@ do
if [ "$format" = "openfst-tropical" ]; then
for testfile in merge_weighted
do
- rm -f result result1 result2
- if ! (ls $testfile.xfst 2> /dev/null); then
+ ${REMOVE} result result1 result2
+ if ! (${LS} $testfile.xfst 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
- if ! (cat $testfile.xfst | ../hfst-xfst --pipe-mode -q -f $format > result 2> /dev/null); then
+ if ! (${CAT} $testfile.xfst | ../hfst-xfst --pipe-mode=input -q -f $format > result 2> /dev/null); then
echo "ERROR: in compiling "$testfile".xfst"
exit 1;
fi
- if ! (cat result | ${TXT2FST} > tmp1; cat $testfile.att | ${TXT2FST} > tmp2; ); then
+ if ! (${CAT} result | ${TXT2FST} > tmp1; ${CAT} $testfile.att | ${TXT2FST} > tmp2; ); then
echo "ERROR: in compiling "$testfile".att"
exit 1;
fi
@@ -170,64 +181,64 @@ do
shortest_string set_variable info print_net eliminate_flag empty_context xerox_composition \
compile_replace_1 compile_replace_2 compile_replace_3
do
- if ! (ls $testfile.xfst 2> /dev/null); then
+ if ! (${LS} $testfile.xfst 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
# apply up/down leak to stdout with readline..
- if ! (cat $testfile.xfst | ../hfst-xfst --pipe-mode -f $format -s | tr -d '\r' > tmp); then
+ if ! (${CAT} $testfile.xfst | ../hfst-xfst --pipe-mode=input -f $format -s | ${TR} -d '\r' > tmp); then
echo "ERROR: in compiling "$testfile.xfst
exit 1;
fi
if ! ($DIFF tmp $testfile.output > tmpdiff); then
- if (ls $testfile.alternative_output > /dev/null 2> /dev/null); then
+ if (${LS} $testfile.alternative_output > /dev/null 2> /dev/null); then
if ! ($DIFF tmp $testfile.alternative_output); then
- rm -f tmpdiff
+ ${REMOVE} tmpdiff
echo "ERROR: in result from "$testfile.xfst
exit 1;
fi
else
- cat tmpdiff
- rm -f tmpdiff
+ ${CAT} tmpdiff
+ ${REMOVE} tmpdiff
echo "ERROR: in result from "$testfile.xfst
exit 1;
fi
- rm -f tmpdiff
+ ${REMOVE} tmpdiff
fi
done
## Interactive commands
for testfile in apply_up apply_down inspect_net
do
- if ! (ls $testfile.xfst 2> /dev/null); then
+ if ! (${LS} $testfile.xfst 2> /dev/null); then
echo "skipping missing test for "$testfile"..."
continue
fi
# apply up/down leak to stdout with readline..
- for param in --pipe-mode # --no-readline
+ for param in --pipe-mode=input # --no-readline
do
# 'inspect net' requires input from stdin
- if (test "$param" = "--pipe-mode" -a "$testfile" = "inspect_net"); then
+ if (${TEST} "$param" = "--pipe-mode=input" -a "$testfile" = "inspect_net"); then
continue
fi
- if ! (cat $testfile.xfst | ../hfst-xfst $param -f $format -s | tr -d '\r' > tmp); then
+ if ! (${CAT} $testfile.xfst | ../hfst-xfst $param -f $format -s | ${TR} -d '\r' > tmp); then
echo "ERROR: in compiling "$testfile.xfst" with parameters "$param
exit 1;
fi
if ! ($DIFF tmp $testfile.output > tmpdiff); then
- if (ls $testfile.alternative_output > /dev/null 2> /dev/null); then
+ if (${LS} $testfile.alternative_output > /dev/null 2> /dev/null); then
if ! ($DIFF tmp $testfile.alternative_output); then
- rm -f tmpdiff
+ ${REMOVE} tmpdiff
echo "ERROR: in result from "$testfile.xfst
exit 1;
fi
else
- cat tmpdiff
- rm -f tmpdiff
+ ${CAT} tmpdiff
+ ${REMOVE} tmpdiff
echo "ERROR: in result from "$testfile.xfst
exit 1;
fi
- rm -f tmpdiff
+ ${REMOVE} tmpdiff
fi
done
done
@@ -239,11 +250,11 @@ do
do
for testfile in test_overlap test_sublanguage # the function to be tested
do
- if ! (ls $testfile$testcase.xfst 2> /dev/null); then
+ if ! (${LS} $testfile$testcase.xfst 2> /dev/null); then
echo "skipping missing test for "$testfile$testcase"..."
continue
fi
- if ! (cat $testfile$testcase.xfst | ../hfst-xfst --pipe-mode -s -f $format | tr -d '\r' > tmp); then
+ if ! (${CAT} $testfile$testcase.xfst | ../hfst-xfst --pipe-mode=input -s -f $format | ${TR} -d '\r' > tmp); then
echo "ERROR: in compiling "$testfile$testcase.xfst
exit 1;
fi
@@ -256,15 +267,15 @@ do
for file in quit-on-fail.xfst assert.xfst
do
- if (cat $file | ../hfst-xfst --pipe-mode -s -f $format > tmp 2> /dev/null); then
+ if (${CAT} $file | ../hfst-xfst --pipe-mode=input -s -f $format > tmp 2> /dev/null); then
echo "ERROR: in compiling "$file
exit 1;
fi
- if (grep '^fail' tmp > /dev/null); then
+ if (${GREP} '^fail' tmp > /dev/null); then
echo "ERROR: in testing "$file
exit 1;
fi
- if ! (grep '^pass' tmp > /dev/null); then
+ if ! (${GREP} '^pass' tmp > /dev/null); then
echo "ERROR: in testing "$file
exit 1;
fi
@@ -284,19 +295,19 @@ do
weighted_ltr_longest_match_1 weighted_ltr_longest_match_2 weighted_ltr_longest_match_3 \
weighted_ltr_shortest_match_1 weighted_ltr_shortest_match_2 weighted_ltr_shortest_match_3
do
- if ! (ls $file.xfst 2> /dev/null); then
+ if ! (${LS} $file.xfst 2> /dev/null); then
echo "skipping missing test for "$file"..."
continue
fi
- if ! (cat $file.xfst | ../hfst-xfst --pipe-mode -s -f $format | tr -d '\r' > tmp 2> /dev/null); then
+ if ! (${CAT} $file.xfst | ../hfst-xfst --pipe-mode=input -s -f $format | ${TR} -d '\r' > tmp 2> /dev/null); then
echo "ERROR: in compiling "$file".xfst"
exit 1;
fi
- if ! (cat tmp | ${STRINGS2FST} -j -f $format > tmp1 2> /dev/null); then
+ if ! (${CAT} tmp | ${STRINGS2FST} -j -f $format > tmp1 2> /dev/null); then
echo "ERROR: in compiling result file from "$file".xfst"
exit 1;
fi
- if ! (cat $file.result | ${STRINGS2FST} -j -f $format > tmp2 2> /dev/null); then
+ if ! (${CAT} $file.result | ${STRINGS2FST} -j -f $format > tmp2 2> /dev/null); then
echo "ERROR: in compiling file "$file".result"
exit 1;
fi
@@ -322,22 +333,22 @@ do
weighted_parallel_rules_7 weighted_parallel_rules_8 weighted_parallel_rules_9 \
weighted_parallel_rules_10 weighted_parallel_rules_11
do
- if ! (ls $file.xfst 2> /dev/null); then
+ if ! (${LS} $file.xfst 2> /dev/null); then
echo "skipping missing test for "$file"..."
continue
fi
- if ! (cat $file.xfst | ../hfst-xfst --pipe-mode -s -f $format | tr -d '\r' > tmp 2> /dev/null); then
+ if ! (${CAT} $file.xfst | ../hfst-xfst --pipe-mode=input -s -f $format | ${TR} -d '\r' > tmp 2> /dev/null); then
echo "ERROR: in compiling "$file".xfst"
exit 1;
fi
- if ! (diff $file.output tmp); then
+ if ! (${DIFF} $file.output tmp); then
echo "ERROR: "$file" test failed"
exit 1;
fi
done
fi
- rm -f result tmp1 tmp2 foo
+ ${REMOVE} result tmp1 tmp2 foo
## add properties
# alias
diff --git a/tools/src/parsers/xfst-lexer.ll b/tools/src/parsers/xfst-lexer.ll
index 614c725..e41a832 100644
--- a/tools/src/parsers/xfst-lexer.ll
+++ b/tools/src/parsers/xfst-lexer.ll
@@ -663,13 +663,14 @@ LWSP [\t ]*
unsigned int endpos = (unsigned int)end_found;
// copy the input to apply and set is as return value
- char buf [endpos + 1];
+ char * buf = (char*) malloc(endpos + 1);
for (unsigned int i=0; i < endpos; i++)
{
buf[i] = hxfsttext[i];
}
buf[endpos] = '\0';
hxfstlval.text = strdup(buf);
+ free(buf);
// put back the rest of the input text, excluding the "<ctrl-d>"
if (total_length > 0)
@@ -704,13 +705,14 @@ LWSP [\t ]*
(void) hfst::xfst::xfst_->compile_regex(hxfsttext, chars_read);
// copy the input to regex and set is as return value
- char buf [chars_read+1];
+ char * buf = (char*) malloc(chars_read+1);
for (unsigned int i=0; i < chars_read; i++)
{
buf[i] = hxfsttext[i];
}
buf[chars_read] = '\0';
hxfstlval.text = strdup(buf);
+ free(buf);
// put back the rest of the input text
if (total_length > 0)
diff --git a/tools/src/parsers/xfst-parser.yy b/tools/src/parsers/xfst-parser.yy
index 619f4c4..379e44c 100644
--- a/tools/src/parsers/xfst-parser.yy
+++ b/tools/src/parsers/xfst-parser.yy
@@ -62,7 +62,7 @@ int hxfstlex(void);
DEFINE_NAME DEFINE_FUNCTION
%token <list> RANGE
%token <file> REDIRECT_IN REDIRECT_OUT
-%token SAVE_PROLOG LOWER FOR REVERSE VIEW LOADD PRINT_LABEL_COUNT
+%token SAVE_PROLOG FOR REVERSE VIEW LOADD PRINT_LABEL_COUNT
TEST_OVERLAP TEST_NONNULL CONCATENATE LOADS INVERT PRINT_ALIASES
PRINT_LABELS XFST_OPTIONAL PRINT_SHORTEST_STRING_SIZE READ_PROPS
TEST_FUNCT PRINT_LABELMAPS SUBSTRING COMPOSE READ_SPACED
@@ -93,7 +93,7 @@ int hxfstlex(void);
%token <text> REGEX
%token <text> APPLY_INPUT
-%type <text> COMMAND_SEQUENCE NAMETOKEN_LIST LABEL_LIST LABEL
+%type <text> COMMAND_SEQUENCE NAMETOKEN_LIST QUOTED_NAMETOKEN_LIST LABEL_LIST LABEL
%%
XFST_SCRIPT: COMMAND_LIST
@@ -430,7 +430,7 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
free($2);
free($4);
}
- | SUBSTITUTE_SYMBOL NAMETOKEN_LIST FOR NAMETOKEN END_COMMAND {
+ | SUBSTITUTE_SYMBOL QUOTED_NAMETOKEN_LIST FOR NAMETOKEN END_COMMAND {
hfst::xfst::xfst_->substitute_symbol($2, $4);
free($2);
free($4);
@@ -1184,6 +1184,50 @@ NAMETOKEN_LIST: NAMETOKEN_LIST NAMETOKEN {
}
;
+QUOTED_NAMETOKEN_LIST: QUOTED_NAMETOKEN_LIST NAMETOKEN {
+ $$ = static_cast<char*>(malloc(sizeof(char)*strlen($1)+strlen($2)+4));
+ char* s = $1;
+ char* r = $$;
+ while (*s != '\0')
+ {
+ *r = *s;
+ r++;
+ s++;
+ }
+ *r = ' ';
+ r++;
+ s = $2;
+ *r = '"';
+ r++;
+ while (*s != '\0')
+ {
+ *r = *s;
+ r++;
+ s++;
+ }
+ *r = '"';
+ r++;
+ *r = '\0';
+ }
+ | NAMETOKEN {
+ $$ = static_cast<char*>(malloc(sizeof(char)*strlen($1)+3));
+ char* s = $1;
+ char* r = $$;
+ *r = '"';
+ r++;
+ while (*s != '\0')
+ {
+ *r = *s;
+ r++;
+ s++;
+ }
+ *r = '"';
+ r++;
+ *r = '\0';
+ free($1);
+ }
+ ;
+
LABEL: NAMETOKEN COLON NAMETOKEN {
$$ = strdup((std::string($1) + std::string(":") + std::string($3)).c_str());
}
diff --git a/tools/src/parsers/xfst-utils.h b/tools/src/parsers/xfst-utils.h
index b134b9d..3a022e6 100644
--- a/tools/src/parsers/xfst-utils.h
+++ b/tools/src/parsers/xfst-utils.h
@@ -24,6 +24,12 @@
#endif
#include <string>
+#ifdef _MSC_VER
+# include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#endif
+
+
namespace hfst { namespace xfst {
#ifndef HAVE_GETLINE
diff --git a/tools/src/parsers/help_message.cc b/tools/src/parsers/xfst_help_message.cc
similarity index 99%
rename from tools/src/parsers/help_message.cc
rename to tools/src/parsers/xfst_help_message.cc
index cbc85a2..246d68d 100644
--- a/tools/src/parsers/help_message.cc
+++ b/tools/src/parsers/xfst_help_message.cc
@@ -1,3 +1,6 @@
+#include <string>
+#include <vector>
+
namespace hfst
{
namespace xfst
diff --git a/tools/src/parsers/xfst_help_message.h b/tools/src/parsers/xfst_help_message.h
new file mode 100644
index 0000000..1ee475d
--- /dev/null
+++ b/tools/src/parsers/xfst_help_message.h
@@ -0,0 +1,61 @@
+namespace hfst
+{
+namespace xfst
+{
+
+typedef std::vector<std::string> StringVector;
+
+// find help message for a certain command
+#define HELP_MODE_ONE_COMMAND 0
+// get all commands and their help messages
+#define HELP_MODE_ALL_COMMANDS 1
+// get all command-help message pairs, where a certain word is found
+#define HELP_MODE_APROPOS 2
+
+// Convert \a str to upper case.
+ std::string to_upper_case(const std::string & str);
+
+// Whether \a c is a punctuation character.
+ bool is_punctuation_char(char c);
+
+// Whether word \a str_ is found in text \a text_.
+// Punctuation characters and upper/lower case are handled in this function.
+ bool word_found_in_text(const std::string & str_, const std::string & text_);
+
+
+// Convert the list of names separated by commas \a namelist into a vector of names.
+ StringVector namelist_to_name_vector(const std::string & namelist);
+
+// Append help message for command known by names in \a namelist, taking arguments
+// listed in \a arguments, and described by \a description to \a message.
+// \a all_names defines whether all names in \a namelist are included or just the first one.
+void append_help_message(const std::string & namelist, const std::string & arguments,
+ const std::string & description, std::string & message, bool all_names = true);
+
+ bool text_matches_some_name(const std::string & text, const std::string & namelist);
+
+ bool get_help_message(const std::string & text, std::string & message, int help_mode, bool skip_ambiguous_cases=false);
+
+bool handle_ambiguous_case(const std::string & name, const std::string & namelist,
+ const std::string & text, std::string & message, int help_mode);
+
+// If \a text matches (depending on \a help_mode) a command known by names
+// listed in \a names, taking arguments \a arguments, described by \a description,
+// add help message for the command to \a message. \a all_names defines whether
+// all names in \a names are included or just the first one.
+// Return whether the search should continue (depends on \a help_mode).
+bool handle_case(const std::string & names, const std::string & arguments,
+ const std::string & description, const std::string & text,
+ std::string & message, int help_mode, bool all_names=true);
+
+// Generate help message(s) for command(s) named \a text and append the help message(s)
+// to \a message. \a help_mode defines whether we are generating help messages for \a text,
+// all commands (in that case, \a message is ignored) or for commands that contain or
+// whose help messages contain the word \a text. \a skip_ambiguous_cases defines whether
+// ambiguous cases where \a text matches more than one command are ignored.
+// @return Whether the help message could be generated.
+ bool get_help_message(const std::string & text, std::string & message, int help_mode,
+ bool skip_ambiguous_cases);
+
+}
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/hfst.git
More information about the debian-science-commits
mailing list