NIFI-5826 Fix back-slash escaping at Lexers
authorKoji Kawamura <ijokarumawak@apache.org>
Wed, 5 Dec 2018 06:03:21 +0000 (15:03 +0900)
committerEd <edward.berezitsky@gmail.com>
Tue, 8 Jan 2019 21:50:20 +0000 (16:50 -0500)
Adding unit test cases for escaped characters

Signed-off-by: Ed <edward.berezitsky@gmail.com>
This closes #3200

nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
nifi-commons/nifi-hl7-query-language/src/main/antlr3/org/apache/nifi/hl7/query/antlr/HL7QueryLexer.g
nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathLexer.g
nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java

index 6c0bcff..fb3958c 100644 (file)
@@ -232,7 +232,7 @@ ESC
                        |       '\\'    { setText("\\\\"); }
                        |       nextChar = ~('"' | '\'' | 'r' | 'n' | 't' | '\\')
                                {
-                                       StringBuilder lBuf = new StringBuilder(); lBuf.append("\\\\").appendCodePoint(nextChar); setText(lBuf.toString());
+                                       StringBuilder lBuf = new StringBuilder(); lBuf.append("\\").appendCodePoint(nextChar); setText(lBuf.toString());
                                }
                )
        ;
index 478028b..ab9d355 100644 (file)
@@ -162,7 +162,7 @@ ESC
                        |       '\\'    { setText("\\\\"); }
                        |       nextChar = ~('"' | '\'' | 'r' | 'n' | 't' | '\\')               
                                {
-                                       StringBuilder lBuf = new StringBuilder(); lBuf.append("\\\\").appendCodePoint(nextChar); setText(lBuf.toString());
+                                       StringBuilder lBuf = new StringBuilder(); lBuf.append("\\").appendCodePoint(nextChar); setText(lBuf.toString());
                                }
                )
        ;
index cd466f7..b6a8380 100644 (file)
@@ -152,7 +152,7 @@ ESC
                        |       '\\'    { setText("\\\\"); }
                        |       nextChar = ~('"' | '\'' | 'r' | 'n' | 't' | '\\')
                                {
-                                       StringBuilder lBuf = new StringBuilder(); lBuf.append("\\\\").appendCodePoint(nextChar); setText(lBuf.toString());
+                                       StringBuilder lBuf = new StringBuilder(); lBuf.append("\\").appendCodePoint(nextChar); setText(lBuf.toString());
                                }
                )
        ;
index 67c14e6..881fb64 100644 (file)
@@ -1026,6 +1026,133 @@ public class TestRecordPath {
         assertEquals("Jxohn Dxoe", RecordPath.compile("replaceRegex(/name, '(?<hello>[JD])', '${hello}x')").evaluate(record).getSelectedFields().findFirst().get().getValue());
 
         assertEquals("48ohn 48oe", RecordPath.compile("replaceRegex(/name, '(?<hello>[JD])', /id)").evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+    }
+
+    @Test
+    public void testReplaceRegexEscapedCharacters() {
+        final List<RecordField> fields = new ArrayList<>();
+        fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+        fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+
+        final RecordSchema schema = new SimpleRecordSchema(fields);
+
+        final Map<String, Object> values = new HashMap<>();
+        values.put("id", 48);
+        final Record record = new MapRecord(schema, values);
+
+        // Special character cases
+        values.put("name", "John Doe");
+        assertEquals("Replacing whitespace to new line",
+                "John\nDoe", RecordPath.compile("replaceRegex(/name, '[\\s]', '\\n')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "John\nDoe");
+        assertEquals("Replacing new line to whitespace",
+                "John Doe", RecordPath.compile("replaceRegex(/name, '\\n', ' ')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "John Doe");
+        assertEquals("Replacing whitespace to tab",
+                "John\tDoe", RecordPath.compile("replaceRegex(/name, '[\\s]', '\\t')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "John\tDoe");
+        assertEquals("Replacing tab to whitespace",
+                "John Doe", RecordPath.compile("replaceRegex(/name, '\\t', ' ')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+    }
+
+    @Test
+    public void testReplaceRegexEscapedQuotes() {
+
+        final List<RecordField> fields = new ArrayList<>();
+        fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+        fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+
+        final RecordSchema schema = new SimpleRecordSchema(fields);
+
+        final Map<String, Object> values = new HashMap<>();
+        values.put("id", 48);
+        final Record record = new MapRecord(schema, values);
+
+        // Quotes
+        // NOTE: At Java code, a single back-slash needs to be escaped with another-back slash, but needn't to do so at NiFi UI.
+        //       The test record path is equivalent to replaceRegex(/name, '\'', '"')
+        values.put("name", "'John' 'Doe'");
+        assertEquals("Replacing quote to double-quote",
+                "\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name, '\\'', '\"')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "\"John\" \"Doe\"");
+        assertEquals("Replacing double-quote to single-quote",
+                "'John' 'Doe'", RecordPath.compile("replaceRegex(/name, '\"', '\\'')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "'John' 'Doe'");
+        assertEquals("Replacing quote to double-quote, the function arguments are wrapped by double-quote",
+                "\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name, \"'\", \"\\\"\")")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "\"John\" \"Doe\"");
+        assertEquals("Replacing double-quote to single-quote, the function arguments are wrapped by double-quote",
+                "'John' 'Doe'", RecordPath.compile("replaceRegex(/name, \"\\\"\", \"'\")")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+    }
+
+    @Test
+    public void testReplaceRegexEscapedBackSlashes() {
+
+        final List<RecordField> fields = new ArrayList<>();
+        fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+        fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+
+        final RecordSchema schema = new SimpleRecordSchema(fields);
+
+        final Map<String, Object> values = new HashMap<>();
+        values.put("id", 48);
+        final Record record = new MapRecord(schema, values);
+
+        // Back-slash
+        // NOTE: At Java code, a single back-slash needs to be escaped with another-back slash, but needn't to do so at NiFi UI.
+        //       The test record path is equivalent to replaceRegex(/name, '\\', '/')
+        values.put("name", "John\\Doe");
+        assertEquals("Replacing a back-slash to forward-slash",
+                "John/Doe", RecordPath.compile("replaceRegex(/name, '\\\\', '/')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "John/Doe");
+        assertEquals("Replacing a forward-slash to back-slash",
+                "John\\Doe", RecordPath.compile("replaceRegex(/name, '/', '\\\\')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+    }
+
+    @Test
+    public void testReplaceRegexEscapedBrackets() {
+
+        final List<RecordField> fields = new ArrayList<>();
+        fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+        fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+
+        final RecordSchema schema = new SimpleRecordSchema(fields);
+
+        final Map<String, Object> values = new HashMap<>();
+        values.put("id", 48);
+        final Record record = new MapRecord(schema, values);
+
+        // Brackets
+        values.put("name", "J[o]hn Do[e]");
+        assertEquals("Square brackets can be escaped with back-slash",
+                "J(o)hn Do(e)", RecordPath.compile("replaceRegex(replaceRegex(/name, '\\[', '('), '\\]', ')')")
+                .evaluate(record).getSelectedFields().findFirst().get().getValue());
+
+        values.put("name", "J(o)hn Do(e)");
+        assertEquals("Brackets can be escaped with back-slash",
+                "J[o]hn Do[e]", RecordPath.compile("replaceRegex(replaceRegex(/name, '\\(', '['), '\\)', ']')")
+                        .evaluate(record).getSelectedFields().findFirst().get().getValue());
     }
 
     @Test