8.11.2

- Use Array.map instead of for-loop
- Add test case

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 All notable changes to CyberChef will be documented in this file.
 
+### [8.11.0] - 2018-11-13
+- 'CSV to JSON' and 'JSON to CSV' operations added [@n1474335] | [#277]
+
+### [8.10.0] - 2018-11-07
+- 'Remove Diacritics' operation added [@klaxon1] | [#387]
+
 ### [8.9.0] - 2018-11-07
 - 'Defang URL' operation added [@arnydo] | [#394]
 
@@ -80,6 +86,7 @@ All notable changes to CyberChef will be documented in this file.
 [@sevzero]: https://github.com/sevzero
 [@PenguinGeorge]: https://github.com/PenguinGeorge
 [@arnydo]: https://github.com/arnydo
+[@klaxon1]: https://github.com/klaxon1
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -87,6 +94,7 @@ All notable changes to CyberChef will be documented in this file.
 [#239]: https://github.com/gchq/CyberChef/pull/239
 [#248]: https://github.com/gchq/CyberChef/pull/248
 [#255]: https://github.com/gchq/CyberChef/issues/255
+[#277]: https://github.com/gchq/CyberChef/issues/277
 [#281]: https://github.com/gchq/CyberChef/pull/281
 [#284]: https://github.com/gchq/CyberChef/pull/284
 [#294]: https://github.com/gchq/CyberChef/pull/294
@@ -99,4 +107,5 @@ All notable changes to CyberChef will be documented in this file.
 [#344]: https://github.com/gchq/CyberChef/pull/344
 [#348]: https://github.com/gchq/CyberChef/pull/348
 [#351]: https://github.com/gchq/CyberChef/pull/351
+[#387]: https://github.com/gchq/CyberChef/pull/387
 [#394]: https://github.com/gchq/CyberChef/pull/394

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.9.2",
+  "version": "8.11.2",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/Utils.mjs

@@ -9,6 +9,7 @@ import moment from "moment-timezone";
 import {fromBase64} from "./lib/Base64";
 import {fromHex} from "./lib/Hex";
 import {fromDecimal} from "./lib/Decimal";
+import {fromBinary} from "./lib/Binary";
 
 
 /**
@@ -298,7 +299,7 @@ class Utils {
      * Accepts hex, Base64, UTF8 and Latin1 strings.
      *
      * @param {string} str
-     * @param {string} type - One of "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
+     * @param {string} type - One of "Binary", "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
      * @returns {byteArray}
      *
      * @example
@@ -313,6 +314,8 @@ class Utils {
      */
     static convertToByteArray(str, type) {
         switch (type.toLowerCase()) {
+            case "binary":
+                return fromBinary(str);
             case "hex":
                 return fromHex(str);
             case "decimal":
@@ -333,7 +336,7 @@ class Utils {
      * Accepts hex, Base64, UTF8 and Latin1 strings.
      *
      * @param {string} str
-     * @param {string} type - One of "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
+     * @param {string} type - One of "Binary", "Hex", "Decimal", "Base64", "UTF8" or "Latin1"
      * @returns {string}
      *
      * @example
@@ -348,6 +351,8 @@ class Utils {
      */
     static convertToByteString(str, type) {
         switch (type.toLowerCase()) {
+            case "binary":
+                return Utils.byteArrayToChars(fromBinary(str));
             case "hex":
                 return Utils.byteArrayToChars(fromHex(str));
             case "decimal":
@@ -568,6 +573,10 @@ class Utils {
                 cell = "";
                 lines.push(line);
                 line = [];
+                // Skip next byte if it is also a line delim (e.g. \r\n)
+                if (lineDelims.indexOf(next) >= 0 && next !== b) {
+                    i++;
+                }
             } else {
                 cell += b;
             }

src/core/config/Categories.json

@@ -54,7 +54,9 @@
             "From MessagePack",
             "To Braille",
             "From Braille",
-            "Parse TLV"
+            "Parse TLV",
+            "CSV to JSON",
+            "JSON to CSV"
         ]
     },
     {
@@ -168,6 +170,7 @@
         "ops": [
             "Encode text",
             "Decode text",
+            "Remove Diacritics",
             "Unescape Unicode Characters"
         ]
     },

src/core/lib/Base64.mjs

@@ -126,14 +126,14 @@ export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", r
  * Base64 alphabets.
  */
 export const ALPHABET_OPTIONS = [
-    {name: "Standard: A-Za-z0-9+/=", value: "A-Za-z0-9+/="},
-    {name: "URL safe: A-Za-z0-9-_", value: "A-Za-z0-9-_"},
+    {name: "Standard (RFC 4648): A-Za-z0-9+/=", value: "A-Za-z0-9+/="},
+    {name: "URL safe (RFC 4648 \u00A75): A-Za-z0-9-_", value: "A-Za-z0-9-_"},
     {name: "Filename safe: A-Za-z0-9+-=", value: "A-Za-z0-9+\\-="},
     {name: "itoa64: ./0-9A-Za-z=", value: "./0-9A-Za-z="},
     {name: "XML: A-Za-z0-9_.", value: "A-Za-z0-9_."},
     {name: "y64: A-Za-z0-9._-", value: "A-Za-z0-9._-"},
     {name: "z64: 0-9a-zA-Z+/=", value: "0-9a-zA-Z+/="},
-    {name: "Radix-64: 0-9A-Za-z+/=", value: "0-9A-Za-z+/="},
+    {name: "Radix-64 (RFC 4880): 0-9A-Za-z+/=", value: "0-9A-Za-z+/="},
     {name: "Uuencoding: [space]-_", value: " -_"},
     {name: "Xxencoding: +-0-9A-Za-z", value: "+\\-0-9A-Za-z"},
     {name: "BinHex: !-,-0-689@A-NP-VX-Z[`a-fh-mp-r", value: "!-,-0-689@A-NP-VX-Z[`a-fh-mp-r"},

src/core/lib/Binary.mjs

@@ -0,0 +1,70 @@
+/**
+ * Binary functions.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Utils from "../Utils";
+
+
+/**
+ * Convert a byte array into a binary string.
+ *
+ * @param {Uint8Array|byteArray} data
+ * @param {string} [delim="Space"]
+ * @param {number} [padding=8]
+ * @returns {string}
+ *
+ * @example
+ * // returns "00010000 00100000 00110000"
+ * toBinary([10,20,30]);
+ *
+ * // returns "00010000 00100000 00110000"
+ * toBinary([10,20,30], ":");
+ */
+export function toBinary(data, delim="Space", padding=8) {
+    if (!data) return "";
+
+    delim = Utils.charRep(delim);
+    let output = "";
+
+    for (let i = 0; i < data.length; i++) {
+        output += data[i].toString(2).padStart(padding, "0") + delim;
+    }
+
+    if (delim.length) {
+        return output.slice(0, -delim.length);
+    } else {
+        return output;
+    }
+}
+
+
+/**
+ * Convert a binary string into a byte array.
+ *
+ * @param {string} data
+ * @param {string} [delim]
+ * @param {number} [byteLen=8]
+ * @returns {byteArray}
+ *
+ * @example
+ * // returns [10,20,30]
+ * fromBinary("00010000 00100000 00110000");
+ *
+ * // returns [10,20,30]
+ * fromBinary("00010000:00100000:00110000", "Colon");
+ */
+export function fromBinary(data, delim="Space", byteLen=8) {
+    const delimRegex = Utils.regexRep(delim);
+    data = data.replace(delimRegex, "");
+
+    const output = [];
+    for (let i = 0; i < data.length; i += byteLen) {
+        output.push(parseInt(data.substr(i, byteLen), 2));
+    }
+    return output;
+}
+

src/core/lib/BitwiseOp.mjs

@@ -116,3 +116,9 @@ export function sub(operand, key) {
     const result = operand - key;
     return (result < 0) ? 256 + result : result;
 }
+
+
+/**
+ * Delimiter options for bitwise operations
+ */
+export const BITWISE_OP_DELIMS = ["Hex", "Decimal", "Binary", "Base64", "UTF8", "Latin1"];

src/core/operations/ADD.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, add } from "../lib/BitwiseOp";
+import { bitOp, add, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * ADD operation
@@ -30,7 +30,7 @@ class ADD extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             }
         ];
     }

src/core/operations/AND.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, and } from "../lib/BitwiseOp";
+import { bitOp, and, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * AND operation
@@ -30,7 +30,7 @@ class AND extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             }
         ];
     }

src/core/operations/CSVToJSON.mjs

@@ -0,0 +1,80 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+
+/**
+ * CSV to JSON operation
+ */
+class CSVToJSON extends Operation {
+
+    /**
+     * CSVToJSON constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "CSV to JSON";
+        this.module = "Default";
+        this.description = "Converts a CSV file to JSON format.";
+        this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
+        this.inputType = "string";
+        this.outputType = "JSON";
+        this.args = [
+            {
+                name: "Cell delimiters",
+                type: "binaryShortString",
+                value: ","
+            },
+            {
+                name: "Row delimiters",
+                type: "binaryShortString",
+                value: "\\r\\n"
+            },
+            {
+                name: "Format",
+                type: "option",
+                value: ["Array of dictionaries", "Array of arrays"]
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        const [cellDelims, rowDelims, format] = args;
+        let json, header;
+
+        try {
+            json = Utils.parseCSV(input, cellDelims.split(""), rowDelims.split(""));
+        } catch (err) {
+            throw new OperationError("Unable to parse CSV: " + err);
+        }
+
+        switch (format) {
+            case "Array of dictionaries":
+                header = json[0];
+                return json.slice(1).map(row => {
+                    const obj = {};
+                    header.forEach((h, i) => {
+                        obj[h] = row[i];
+                    });
+                    return obj;
+                });
+            case "Array of arrays":
+            default:
+                return json;
+        }
+    }
+
+}
+
+export default CSVToJSON;

src/core/operations/FromBinary.mjs

@@ -7,6 +7,7 @@
 import Operation from "../Operation";
 import Utils from "../Utils";
 import {BIN_DELIM_OPTIONS} from "../lib/Delim";
+import {fromBinary} from "../lib/Binary";
 
 /**
  * From Binary operation
@@ -77,15 +78,7 @@ class FromBinary extends Operation {
      * @returns {byteArray}
      */
     run(input, args) {
-        const delimRegex = Utils.regexRep(args[0] || "Space");
-        input = input.replace(delimRegex, "");
-
-        const output = [];
-        const byteLen = 8;
-        for (let i = 0; i < input.length; i += byteLen) {
-            output.push(parseInt(input.substr(i, byteLen), 2));
-        }
-        return output;
+        return fromBinary(input, args[0]);
     }
 
     /**

src/core/operations/JSONToCSV.mjs

@@ -0,0 +1,111 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+
+/**
+ * JSON to CSV operation
+ */
+class JSONToCSV extends Operation {
+
+    /**
+     * JSONToCSV constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "JSON to CSV";
+        this.module = "Default";
+        this.description = "Converts JSON data to a CSV based on the definition in RFC 4180.";
+        this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
+        this.inputType = "JSON";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Cell delimiter",
+                type: "binaryShortString",
+                value: ","
+            },
+            {
+                name: "Row delimiter",
+                type: "binaryShortString",
+                value: "\\r\\n"
+            }
+        ];
+    }
+
+    /**
+     * @param {JSON} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const [cellDelim, rowDelim] = args;
+
+        // Record values so they don't have to be passed to other functions explicitly
+        this.cellDelim = cellDelim;
+        this.rowDelim = rowDelim;
+        const self = this;
+
+        try {
+            // If the JSON is an array of arrays, this is easy
+            if (input[0] instanceof Array) {
+                return input
+                    .map(row => row
+                        .map(self.escapeCellContents.bind(self))
+                        .join(cellDelim)
+                    )
+                    .join(rowDelim) +
+                    rowDelim;
+            }
+
+            // If it's an array of dictionaries...
+            const header = Object.keys(input[0]);
+            return header
+                .map(self.escapeCellContents.bind(self))
+                .join(cellDelim) +
+                rowDelim +
+                input
+                    .map(row => header
+                        .map(h => row[h])
+                        .map(self.escapeCellContents.bind(self))
+                        .join(cellDelim)
+                    )
+                    .join(rowDelim) +
+                rowDelim;
+        } catch (err) {
+            throw new OperationError("Unable to parse JSON to CSV: " + err.toString());
+        }
+    }
+
+    /**
+     * Correctly escapes a cell's contents based on the cell and row delimiters.
+     *
+     * @param {string} data
+     * @returns {string}
+     */
+    escapeCellContents(data) {
+        // Double quotes should be doubled up
+        data = data.replace(/"/g, '""');
+
+        // If the cell contains a cell or row delimiter or a double quote, it mut be enclosed in double quotes
+        if (
+            data.indexOf(this.cellDelim) >= 0 ||
+            data.indexOf(this.rowDelim) >= 0 ||
+            data.indexOf("\n") >= 0 ||
+            data.indexOf("\r") >= 0 ||
+            data.indexOf('"') >= 0
+        ) {
+            data = `"${data}"`;
+        }
+
+        return data;
+    }
+
+}
+
+export default JSONToCSV;

src/core/operations/OR.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, or } from "../lib/BitwiseOp";
+import { bitOp, or, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * OR operation
@@ -30,7 +30,7 @@ class OR extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             }
         ];
     }

src/core/operations/RemoveDiacritics.mjs

@@ -0,0 +1,41 @@
+/**
+ * @author Klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * Remove Diacritics operation
+ */
+class RemoveDiacritics extends Operation {
+
+    /**
+     * RemoveDiacritics constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Remove Diacritics";
+        this.module = "Default";
+        this.description = "Replaces accented characters with their latin character equivalent.";
+        this.infoURL = "https://wikipedia.org/wiki/Diacritic";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        // reference: https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript/37511463
+        return input.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+    }
+
+}
+
+export default RemoveDiacritics;

src/core/operations/SUB.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, sub } from "../lib/BitwiseOp";
+import { bitOp, sub, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * SUB operation
@@ -30,7 +30,7 @@ class SUB extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             }
         ];
     }

src/core/operations/Sort.mjs

@@ -38,7 +38,7 @@ class Sort extends Operation {
             {
                 "name": "Order",
                 "type": "option",
-                "value": ["Alphabetical (case sensitive)", "Alphabetical (case insensitive)", "IP address", "Numeric"]
+                "value": ["Alphabetical (case sensitive)", "Alphabetical (case insensitive)", "IP address", "Numeric", "Numeric (hexadecimal)"]
             }
         ];
     }
@@ -62,6 +62,8 @@ class Sort extends Operation {
             sorted = sorted.sort(Sort._ipSort);
         } else if (order === "Numeric") {
             sorted = sorted.sort(Sort._numericSort);
+        } else if (order === "Numeric (hexadecimal)") {
+            sorted = sorted.sort(Sort._hexadecimalSort);
         }
 
         if (sortReverse) sorted.reverse();
@@ -131,6 +133,44 @@ class Sort extends Operation {
         return a.localeCompare(b);
     }
 
+    /**
+     * Comparison operation for sorting of hexadecimal values.
+     *
+     * @author Chris van Marle
+     * @private
+     * @param {string} a
+     * @param {string} b
+     * @returns {number}
+     */
+    static _hexadecimalSort(a, b) {
+        let a_ = a.split(/([^\da-f]+)/i),
+            b_ = b.split(/([^\da-f]+)/i);
+
+        a_ = a_.map(v => {
+            const t = parseInt(v, 16);
+            return isNaN(t) ? v : t;
+        });
+
+        b_ = b_.map(v => {
+            const t = parseInt(v, 16);
+            return isNaN(t) ? v : t;
+        });
+
+        for (let i = 0; i < a_.length && i < b.length; ++i) {
+            if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+            if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+            if (isNaN(a_[i]) && isNaN(b_[i])) {
+                const ret = a_[i].localeCompare(b_[i]); // Compare strings
+                if (ret !== 0) return ret;
+            }
+            if (!isNaN(a_[i]) && !isNaN(a_[i])) { // Compare numbers
+                if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+            }
+        }
+
+        return a.localeCompare(b);
+    }
+
 }
 
 export default Sort;

src/core/operations/ToBinary.mjs

@@ -7,6 +7,7 @@
 import Operation from "../Operation";
 import Utils from "../Utils";
 import {BIN_DELIM_OPTIONS} from "../lib/Delim";
+import {toBinary} from "../lib/Binary";
 
 /**
  * To Binary operation
@@ -40,19 +41,7 @@ class ToBinary extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const delim = Utils.charRep(args[0] || "Space"),
-            padding = 8;
-        let output = "";
-
-        for (let i = 0; i < input.length; i++) {
-            output += input[i].toString(2).padStart(padding, "0") + delim;
-        }
-
-        if (delim.length) {
-            return output.slice(0, -delim.length);
-        } else {
-            return output;
-        }
+        return toBinary(input, args[0]);
     }
 
     /**

src/core/operations/ToTable.mjs

@@ -33,7 +33,7 @@ class ToTable extends Operation {
             {
                 "name": "Row delimiters",
                 "type": "binaryShortString",
-                "value": "\\n\\r"
+                "value": "\\r\\n"
             },
             {
                 "name": "Make first row header",

src/core/operations/XOR.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation";
 import Utils from "../Utils";
-import { bitOp, xor } from "../lib/BitwiseOp";
+import { bitOp, xor, BITWISE_OP_DELIMS } from "../lib/BitwiseOp";
 
 /**
  * XOR operation
@@ -30,7 +30,7 @@ class XOR extends Operation {
                 "name": "Key",
                 "type": "toggleString",
                 "value": "",
-                "toggleValues": ["Hex", "Decimal", "Base64", "UTF8", "Latin1"]
+                "toggleValues": BITWISE_OP_DELIMS
             },
             {
                 "name": "Scheme",

test/index.mjs

@@ -39,6 +39,7 @@ import "./tests/operations/Comment";
 import "./tests/operations/Compress";
 import "./tests/operations/ConditionalJump";
 import "./tests/operations/Crypt";
+import "./tests/operations/CSV";
 import "./tests/operations/DateTime";
 import "./tests/operations/ExtractEmailAddresses";
 import "./tests/operations/Fork";
@@ -63,6 +64,7 @@ import "./tests/operations/ParseIPRange";
 import "./tests/operations/PowerSet";
 import "./tests/operations/Regex";
 import "./tests/operations/Register";
+import "./tests/operations/RemoveDiacritics";
 import "./tests/operations/Rotate";
 import "./tests/operations/SeqUtils";
 import "./tests/operations/SetDifference";
@@ -125,12 +127,12 @@ function handleTestResult(testResult) {
 
 
 /**
- * Fail if the process takes longer than 10 seconds.
+ * Fail if the process takes longer than 60 seconds.
  */
 setTimeout(function() {
-    console.log("Tests took longer than 10 seconds to run, returning.");
+    console.log("Tests took longer than 60 seconds to run, returning.");
     process.exit(1);
-}, 10 * 1000);
+}, 60 * 1000);
 
 
 TestRegister.runTests()

test/tests/operations/CSV.mjs

@@ -0,0 +1,179 @@
+/**
+ * CSV tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ *
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+const EXAMPLE_CSV = `A,B,C,D,E,F\r
+1,2,3,4,5,6\r
+",",;,',"""",,\r
+"""hello""","a""1","multi\r
+line",,,end\r
+`;
+
+TestRegister.addTests([
+    {
+        name: "CSV to JSON: Array of dictionaries",
+        input: EXAMPLE_CSV,
+        expectedOutput: JSON.stringify([
+            {
+                "A": "1",
+                "B": "2",
+                "C": "3",
+                "D": "4",
+                "E": "5",
+                "F": "6"
+            },
+            {
+                "A": ",",
+                "B": ";",
+                "C": "'",
+                "D": "\"",
+                "E": "",
+                "F": ""
+            },
+            {
+                "A": "\"hello\"",
+                "B": "a\"1",
+                "C": "multi\r\nline",
+                "D": "",
+                "E": "",
+                "F": "end"
+            }
+        ], null, 4),
+        recipeConfig: [
+            {
+                op: "CSV to JSON",
+                args: [",", "\r\n", "Array of dictionaries"],
+            }
+        ],
+    },
+    {
+        name: "CSV to JSON: Array of arrays",
+        input: EXAMPLE_CSV,
+        expectedOutput: JSON.stringify([
+            [
+                "A",
+                "B",
+                "C",
+                "D",
+                "E",
+                "F"
+            ],
+            [
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6"
+            ],
+            [
+                ",",
+                ";",
+                "'",
+                "\"",
+                "",
+                ""
+            ],
+            [
+                "\"hello\"",
+                "a\"1",
+                "multi\r\nline",
+                "",
+                "",
+                "end"
+            ]
+        ], null, 4),
+        recipeConfig: [
+            {
+                op: "CSV to JSON",
+                args: [",", "\r\n", "Array of arrays"],
+            }
+        ],
+    },
+    {
+        name: "JSON to CSV: Array of dictionaries",
+        input: JSON.stringify([
+            {
+                "A": "1",
+                "B": "2",
+                "C": "3",
+                "D": "4",
+                "E": "5",
+                "F": "6"
+            },
+            {
+                "A": ",",
+                "B": ";",
+                "C": "'",
+                "D": "\"",
+                "E": "",
+                "F": ""
+            },
+            {
+                "A": "\"hello\"",
+                "B": "a\"1",
+                "C": "multi\r\nline",
+                "D": "",
+                "E": "",
+                "F": "end"
+            }
+        ]),
+        expectedOutput: EXAMPLE_CSV,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\r\n"],
+            }
+        ],
+    },
+    {
+        name: "JSON to CSV: Array of arrays",
+        input: JSON.stringify([
+            [
+                "A",
+                "B",
+                "C",
+                "D",
+                "E",
+                "F"
+            ],
+            [
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6"
+            ],
+            [
+                ",",
+                ";",
+                "'",
+                "\"",
+                "",
+                ""
+            ],
+            [
+                "\"hello\"",
+                "a\"1",
+                "multi\r\nline",
+                "",
+                "",
+                "end"
+            ]
+        ]),
+        expectedOutput: EXAMPLE_CSV,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\r\n"],
+            }
+        ],
+    },
+]);

test/tests/operations/RemoveDiacritics.mjs

@@ -0,0 +1,22 @@
+/**
+ * Remove Diacritics tests.
+ *
+ * @author Klaxon [klaxon@veyr.com]
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Remove Diacritics",
+        input: "\xe0, \xe8, \xec, \xf2, \xf9  \xc0, \xc8, \xcc, \xd2, \xd9\n\xe1, \xe9, \xed, \xf3, \xfa, \xfd \xc1, \xc9, \xcd, \xd3, \xda, \xdd\n\xe2, \xea, \xee, \xf4, \xfb \xc2, \xca, \xce, \xd4, \xdb\n\xe3, \xf1, \xf5 \xc3, \xd1, \xd5\n\xe4, \xeb, \xef, \xf6, \xfc, \xff \xc4, \xcb, \xcf, \xd6, \xdc, \u0178\n\xe5, \xc5",
+        expectedOutput: "a, e, i, o, u  A, E, I, O, U\na, e, i, o, u, y A, E, I, O, U, Y\na, e, i, o, u A, E, I, O, U\na, n, o A, N, O\na, e, i, o, u, y A, E, I, O, U, Y\na, A",
+        recipeConfig: [
+            {
+                "op": "Remove Diacritics",
+                "args": []
+            },
+        ],
+    },
+]);

test/tests/operations/SeqUtils.mjs

@@ -30,4 +30,15 @@ TestRegister.addTests([
             }
         ],
     },
+    {
+        name: "SeqUtils - Hexadecimal sort",
+        input: "06,08,0a,0d,0f,1,10,11,12,13,14,15,16,17,18,19,1a,1b,1c,1d,1e,1f,2,3,4,5,7,9,b,c,e",
+        expectedOutput: "1,2,3,4,5,06,7,08,9,0a,b,c,0d,e,0f,10,11,12,13,14,15,16,17,18,19,1a,1b,1c,1d,1e,1f",
+        recipeConfig: [
+            {
+                "op": "Sort",
+                "args": ["Comma", false, "Numeric (hexadecimal)"]
+            }
+        ],
+    },
 ]);