[jruby-joni] 255/279: Support for \X

Hideki Yamane henrich at moszumanska.debian.org
Mon Nov 16 11:27:46 UTC 2015


This is an automated email from the git hooks/post-receive script.

henrich pushed a commit to branch debian/sid
in repository jruby-joni.

commit 061676e4a115849e6ca1cc8abbb9dd30d5b57fa8
Author: Marcin Mielzynski <lopx at gazeta.pl>
Date:   Fri Apr 10 01:17:53 2015 +0200

    Support for \X
---
 src/org/joni/Lexer.java                            |  3 ++
 src/org/joni/Parser.java                           | 30 +++++++++++
 src/org/joni/Syntax.java                           |  5 +-
 src/org/joni/constants/TokenType.java              |  1 +
 test/org/joni/test/TestA.java                      |  2 +
 .../org/joni/test/TestAU.java                      | 61 ++++++++++++----------
 test/org/joni/test/TestJoni.java                   |  3 ++
 test/org/joni/test/TestU8.java                     |  2 +
 8 files changed, 76 insertions(+), 31 deletions(-)

diff --git a/src/org/joni/Lexer.java b/src/org/joni/Lexer.java
index 2287e77..f447059 100644
--- a/src/org/joni/Lexer.java
+++ b/src/org/joni/Lexer.java
@@ -1125,6 +1125,9 @@ class Lexer extends ScannerSupport {
                 case 'R':
                     if (syntax.op2EscCapitalRLinebreak()) token.type = TokenType.LINEBREAK;
                     break;
+                case 'X':
+                    if (syntax.op2EscCapitalXExtendedGraphemeCluster()) token.type = TokenType.EXTENDED_GRAPHEME_CLUSTER;
+                    break;
                 default:
                     unfetch();
                     int num = fetchEscapedValue();
diff --git a/src/org/joni/Parser.java b/src/org/joni/Parser.java
index 2f01a1c..3891416 100644
--- a/src/org/joni/Parser.java
+++ b/src/org/joni/Parser.java
@@ -27,6 +27,8 @@ import static org.joni.Option.isIgnoreCase;
 import org.jcodings.Ptr;
 import org.jcodings.constants.CharacterType;
 import org.jcodings.constants.PosixBracket;
+import org.jcodings.specific.ASCIIEncoding;
+import org.jcodings.unicode.UnicodeEncoding;
 import org.joni.ast.AnchorNode;
 import org.joni.ast.AnyCharNode;
 import org.joni.ast.BackRefNode;
@@ -759,6 +761,34 @@ class Parser extends Lexer {
             node = en;
             break;
 
+        case EXTENDED_GRAPHEME_CLUSTER:
+            if (Config.USE_UNICODE_PROPERTIES) {
+                if (enc instanceof UnicodeEncoding) {
+                    int ctype = enc.propertyNameToCType(new byte[]{(byte)'M'}, 0, 1);
+                    if (ctype > 0) {
+                        CClassNode cc1 = new CClassNode(); /* \P{M} */
+                        cc1.addCType(ctype, false, env, this);
+                        cc1.setNot();
+                        CClassNode cc2 = new CClassNode(); /* \p{M}* */
+                        cc1.addCType(ctype, false, env, this);
+                        QuantifierNode qn = new QuantifierNode(0, QuantifierNode.REPEAT_INFINITE, false);
+                        qn.setTarget(cc2);
+                        /* \P{M}\p{M}* */
+                        ConsAltNode list2 = ConsAltNode.newListNode(qn, null);
+                        ConsAltNode list1 = ConsAltNode.newListNode(cc1, list2);
+                        EncloseNode en2 = new EncloseNode(EncloseType.STOP_BACKTRACK);
+                        en2.setTarget(list1);
+                        node = en2;
+                    }
+                }
+            }
+            if (node == null) {
+                AnyCharNode np1 = new AnyCharNode();
+                EncloseNode on = new EncloseNode(bsOnOff(env.option, Option.MULTILINE, false), 0);
+                on.setTarget(np1);
+                node = np1;
+            }
+            break;
         case STRING:
             return parseExpTkByte(group); // tk_byte:
 
diff --git a/src/org/joni/Syntax.java b/src/org/joni/Syntax.java
index baea7c6..47f069d 100644
--- a/src/org/joni/Syntax.java
+++ b/src/org/joni/Syntax.java
@@ -420,8 +420,9 @@ public final class Syntax implements SyntaxProperties{
         OP2_CCLASS_SET_OP | OP2_ESC_CAPITAL_C_BAR_CONTROL |
         OP2_ESC_CAPITAL_M_BAR_META | OP2_ESC_V_VTAB |
         OP2_ESC_H_XDIGIT |
-        OP2_ESC_CAPITAL_R_LINEBREAK |
-        OP2_QMARK_LPAREN_CONDITION),
+        OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER |
+        OP2_QMARK_LPAREN_CONDITION |
+        OP2_ESC_CAPITAL_R_LINEBREAK),
 
         ( GNU_REGEX_BV |
         ALLOW_INTERVAL_LOW_ABBREV |
diff --git a/src/org/joni/constants/TokenType.java b/src/org/joni/constants/TokenType.java
index f9a18bd..3dc8386 100644
--- a/src/org/joni/constants/TokenType.java
+++ b/src/org/joni/constants/TokenType.java
@@ -40,6 +40,7 @@ public enum TokenType {
       QUOTE_OPEN,
       CHAR_PROPERTY,    /* \p{...}, \P{...} */
       LINEBREAK,
+      EXTENDED_GRAPHEME_CLUSTER,
       /* in cc */
       CC_CLOSE,
       CC_RANGE,
diff --git a/test/org/joni/test/TestA.java b/test/org/joni/test/TestA.java
index 7b5f689..de5e3b5 100644
--- a/test/org/joni/test/TestA.java
+++ b/test/org/joni/test/TestA.java
@@ -526,6 +526,8 @@ public class TestA extends Test {
         x2s("\\R", "\n", 0, 1);
         x2s("\\R", "\r", 0, 1);
         x2s("\\R{3}", "\r\r\n\n", 0, 4);
+
+        x2s("\\X{5}", "あいab\n", 0, 5);
     }
 
     public static void main(String[] args) throws Throwable{
diff --git a/src/org/joni/constants/TokenType.java b/test/org/joni/test/TestAU.java
similarity index 62%
copy from src/org/joni/constants/TokenType.java
copy to test/org/joni/test/TestAU.java
index f9a18bd..0d37a88 100644
--- a/src/org/joni/constants/TokenType.java
+++ b/test/org/joni/test/TestAU.java
@@ -17,33 +17,36 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-package org.joni.constants;
-
-public enum TokenType {
-      EOT,            /* end of token */
-      RAW_BYTE,
-      CHAR,
-      STRING,
-      CODE_POINT,
-      ANYCHAR,
-      CHAR_TYPE,
-      BACKREF,
-      CALL,
-      ANCHOR,
-      OP_REPEAT,
-      INTERVAL,
-      ANYCHAR_ANYTIME,  /* SQL '%' == .* */
-      ALT,
-      SUBEXP_OPEN,
-      SUBEXP_CLOSE,
-      CC_OPEN,
-      QUOTE_OPEN,
-      CHAR_PROPERTY,    /* \p{...}, \P{...} */
-      LINEBREAK,
-      /* in cc */
-      CC_CLOSE,
-      CC_RANGE,
-      POSIX_BRACKET_OPEN,
-      CC_AND,             /* && */
-      CC_CC_OPEN          /* [ */
+package org.joni.test;
+
+import org.jcodings.Encoding;
+import org.jcodings.specific.ASCIIEncoding;
+import org.joni.Option;
+import org.joni.Syntax;
+
+public class TestAU extends Test {
+
+    public int option() {
+        return Option.DEFAULT;
+    }
+
+    public Encoding encoding() {
+        return ASCIIEncoding.INSTANCE;
+    }
+
+    public String testEncoding() {
+        return "utf-8";
+    }
+
+    public Syntax syntax() {
+        return Syntax.DEFAULT;
+    }
+
+    public void test() throws InterruptedException {
+        x2s("\\X{5}", "あいab\n", 0, 5);
+    }
+
+    public static void main(String[] args) throws Throwable {
+        new TestU8().run();
+    }
 }
diff --git a/test/org/joni/test/TestJoni.java b/test/org/joni/test/TestJoni.java
index d10924d..d7f4fe6 100644
--- a/test/org/joni/test/TestJoni.java
+++ b/test/org/joni/test/TestJoni.java
@@ -31,6 +31,7 @@ public class TestJoni extends TestCase {
     private Test testu8;
     private Test testInterrupt;
     private Test testPerl;
+    private Test testAsciiViaUtf;
 
     protected void setUp() {
         testa = new TestA();
@@ -41,6 +42,7 @@ public class TestJoni extends TestCase {
         testLookBehind = new TestLookBehind();
         testInterrupt = new TestInterrupt();
         testPerl = new TestPerl();
+        testAsciiViaUtf = new TestAU();
     }
 
     protected void tearDown() {
@@ -54,6 +56,7 @@ public class TestJoni extends TestCase {
 
     public void testAscii() {
         testJoniTest(testa);
+        testJoniTest(testAsciiViaUtf);
     }
 
     public void testEUCJP() {
diff --git a/test/org/joni/test/TestU8.java b/test/org/joni/test/TestU8.java
index 86c2812..9723100 100644
--- a/test/org/joni/test/TestU8.java
+++ b/test/org/joni/test/TestU8.java
@@ -84,6 +84,8 @@ public class TestU8 extends Test {
         x2s("\\R", "\u0085", 0, 2);
         x2s("\\R", "\u2028", 0, 3);
         x2s("\\R", "\u2029", 0, 3);
+
+        x2s("\\X", "\u306F\u309A\n", 0, 3);
     }
 
     public static void main(String[] args) throws Throwable {

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-java/jruby-joni.git



More information about the pkg-java-commits mailing list