8.32.1

CHANGELOG.md

@@ -2,6 +2,9 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.32.0] - 2019-06-27
+- 'Indec of Coincidence' operation added [@Ge0rg3] | [#571]
+
 ### [8.31.0] - 2019-04-12
 - The downloadable version of CyberChef is now a .zip file containing separate modules rather than a single .htm file. It is still completely standalone and will not make any external network requests. This change reduces the complexity of the build process significantly. [@n1474335]
 
@@ -130,6 +133,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.32.0]: https://github.com/gchq/CyberChef/releases/tag/v8.32.0
 [8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0
 [8.30.0]: https://github.com/gchq/CyberChef/releases/tag/v8.30.0
 [8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
@@ -189,6 +193,7 @@ All major and minor version changes will be documented in this file. Details of
 [@Cynser]: https://github.com/Cynser
 [@anthony-arnold]: https://github.com/anthony-arnold
 [@masq]: https://github.com/masq
+[@Ge0rg3]: https://github.com/Ge0rg3
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -232,3 +237,4 @@ All major and minor version changes will be documented in this file. Details of
 [#516]: https://github.com/gchq/CyberChef/pull/516
 [#525]: https://github.com/gchq/CyberChef/pull/525
 [#533]: https://github.com/gchq/CyberChef/pull/533
+[#571]: https://github.com/gchq/CyberChef/pull/571

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.31.7",
+  "version": "8.32.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.31.7",
+  "version": "8.32.1",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/Utils.mjs

@@ -201,11 +201,18 @@ class Utils {
      * Utils.parseEscapedChars("\\n");
      */
     static parseEscapedChars(str) {
-        return str.replace(/(\\)?\\([bfnrtv0'"]|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) {
+        return str.replace(/(\\)?\\([bfnrtv'"]|[0-3][0-7]{2}|[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) {
             if (a === "\\") return "\\"+b;
             switch (b[0]) {
                 case "0":
-                    return "\0";
+                case "1":
+                case "2":
+                case "3":
+                case "4":
+                case "5":
+                case "6":
+                case "7":
+                    return String.fromCharCode(parseInt(b, 8));
                 case "b":
                     return "\b";
                 case "t":

src/core/config/Categories.json

@@ -395,6 +395,7 @@
         "ops": [
             "Entropy",
             "Frequency distribution",
+            "Index of Coincidence",
             "Chi Square",
             "Disassemble x86",
             "Pseudo-Random Number Generator",

src/core/operations/IndexOfCoincidence.mjs

@@ -0,0 +1,107 @@
+/**
+ * @author George O [georgeomnet+cyberchef@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+
+/**
+ * Index of Coincidence operation
+ */
+class IndexOfCoincidence extends Operation {
+
+    /**
+     * IndexOfCoincidence constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Index of Coincidence";
+        this.module = "Default";
+        this.description = "Index of Coincidence (IC) is the probability of two randomly selected characters being the same. This can be used to determine whether text is readable or random, with English text having an IC of around 0.066. IC can therefore be a sound method to automate frequency analysis.";
+        this.infoURL = "https://wikipedia.org/wiki/Index_of_coincidence";
+        this.inputType = "string";
+        this.outputType = "number";
+        this.presentType = "html";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {number}
+     */
+    run(input, args) {
+        const text = input.toLowerCase().replace(/[^a-z]/g, ""),
+            frequencies = new Array(26).fill(0),
+            alphabet = Utils.expandAlphRange("a-z");
+        let coincidence = 0.00,
+            density = 0.00,
+            result = 0.00,
+            i;
+
+        for (i=0; i < alphabet.length; i++) {
+            frequencies[i] = text.count(alphabet[i]);
+        }
+
+        for (i=0; i < frequencies.length; i++) {
+            coincidence += frequencies[i] * (frequencies[i] - 1);
+        }
+
+        density = frequencies.sum();
+
+        // Ensure that we don't divide by 0
+        if (density < 2) density = 2;
+
+        result = coincidence / (density * (density - 1));
+
+        return result;
+    }
+
+    /**
+     * Displays the IC as a scale bar for web apps.
+     *
+     * @param {number} ic
+     * @returns {html}
+     */
+    present(ic) {
+        return `Index of Coincidence: ${ic}
+Normalized: ${ic * 26}
+<br><canvas id='chart-area'></canvas><br>
+- 0 represents complete randomness (all characters are unique), whereas 1 represents no randomness (all characters are identical).
+- English text generally has an IC of between 0.67 to 0.78.
+- 'Random' text is determined by the probability that each letter occurs the same number of times as another.
+
+The graph shows the IC of the input data. A low IC generally means that the text is random, compressed or encrypted.
+
+<script type='application/javascript'>
+  var canvas = document.getElementById("chart-area"),
+      parentRect = canvas.parentNode.getBoundingClientRect(),
+      ic = ${ic};
+
+  canvas.width = parentRect.width * 0.95;
+  canvas.height = parentRect.height * 0.25;
+
+  ic = ic > 0.25 ? 0.25 : ic;
+
+  CanvasComponents.drawScaleBar(canvas, ic, 0.25, [
+    {
+      label: "English text",
+      min: 0.05,
+      max: 0.08
+    },
+    {
+      label: "> 0.25",
+      min: 0.24,
+      max: 0.25
+    }
+  ]);
+</script>
+     `;
+    }
+
+}
+
+export default IndexOfCoincidence;

src/core/operations/JSONToCSV.mjs

@@ -51,6 +51,10 @@ class JSONToCSV extends Operation {
         this.rowDelim = rowDelim;
         const self = this;
 
+        if (!(input instanceof Array)) {
+            input = [input];
+        }
+
         try {
             // If the JSON is an array of arrays, this is easy
             if (input[0] instanceof Array) {
@@ -89,6 +93,8 @@ class JSONToCSV extends Operation {
      * @returns {string}
      */
     escapeCellContents(data) {
+        if (typeof data === "number") data = data.toString();
+
         // Double quotes should be doubled up
         data = data.replace(/"/g, '""');
 

src/core/operations/UnescapeString.mjs

@@ -20,7 +20,7 @@ class UnescapeString extends Operation {
 
         this.name = "Unescape string";
         this.module = "Default";
-        this.description = "Unescapes characters in a string that have been escaped. For example, <code>Don\\'t stop me now</code> becomes <code>Don't stop me now</code>.<br><br>Supports the following escape sequences:<ul><li><code>\\n</code> (Line feed/newline)</li><li><code>\\r</code> (Carriage return)</li><li><code>\\t</code> (Horizontal tab)</li><li><code>\\b</code> (Backspace)</li><li><code>\\f</code> (Form feed)</li><li><code>\\xnn</code> (Hex, where n is 0-f)</li><li><code>\\\\</code> (Backslash)</li><li><code>\\'</code> (Single quote)</li><li><code>\\&quot;</code> (Double quote)</li><li><code>\\unnnn</code> (Unicode character)</li><li><code>\\u{nnnnnn}</code> (Unicode code point)</li></ul>";
+        this.description = "Unescapes characters in a string that have been escaped. For example, <code>Don\\'t stop me now</code> becomes <code>Don't stop me now</code>.<br><br>Supports the following escape sequences:<ul><li><code>\\n</code> (Line feed/newline)</li><li><code>\\r</code> (Carriage return)</li><li><code>\\t</code> (Horizontal tab)</li><li><code>\\b</code> (Backspace)</li><li><code>\\f</code> (Form feed)</li><li><code>\\nnn</code> (Octal, where n is 0-7)</li><li><code>\\xnn</code> (Hex, where n is 0-f)</li><li><code>\\\\</code> (Backslash)</li><li><code>\\'</code> (Single quote)</li><li><code>\\&quot;</code> (Double quote)</li><li><code>\\unnnn</code> (Unicode character)</li><li><code>\\u{nnnnnn}</code> (Unicode code point)</li></ul>";
         this.infoURL = "https://wikipedia.org/wiki/Escape_sequence";
         this.inputType = "string";
         this.outputType = "string";

tests/operations/index.mjs

@@ -49,9 +49,11 @@ import "./tests/Hash";
 import "./tests/HaversineDistance";
 import "./tests/Hexdump";
 import "./tests/Image";
+import "./tests/IndexOfCoincidence";
 import "./tests/Jump";
 import "./tests/JSONBeautify";
 import "./tests/JSONMinify";
+import "./tests/JSONtoCSV";
 import "./tests/JWTDecode";
 import "./tests/JWTSign";
 import "./tests/JWTVerify";

tests/operations/tests/IndexOfCoincidence.mjs

@@ -0,0 +1,22 @@
+/**
+ * Index of Coincidence tests.
+ *
+ * @author George O [georgeomnet+cyberchef@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Index of Coincidence",
+        input: "Hello world, this is a test to determine the correct IC value.",
+        expectedMatch: /^Index of Coincidence: 0\.07142857142857142\nNormalized: 1\.857142857142857/,
+        recipeConfig: [
+            {
+                "op": "Index of Coincidence",
+                "args": []
+            },
+        ],
+    },
+]);

tests/operations/tests/JSONtoCSV.mjs

@@ -0,0 +1,93 @@
+/**
+ * JSON to CSV tests.
+ *
+ * @author mshwed [m@ttshwed.com]
+ *
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+const EXPECTED_CSV_SINGLE = "a,b,c\r\n1,2,3\r\n";
+const EXPECTED_CSV_MULTIPLE = "a,b,c\r\n1,2,3\r\n1,2,3\r\n";
+const EXPECTED_CSV_EMPTY = "\r\n\r\n";
+
+TestRegister.addTests([
+    {
+        name: "JSON to CSV: strings as values",
+        input: JSON.stringify({a: "1", b: "2", c: "3"}),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: numbers as values",
+        input: JSON.stringify({a: 1, b: 2, c: 3}),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: numbers and strings as values",
+        input: JSON.stringify({a: 1, b: "2", c: 3}),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: JSON as an array",
+        input: JSON.stringify([{a: 1, b: "2", c: 3}]),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: multiple JSON values in an array",
+        input: JSON.stringify([{a: 1, b: "2", c: 3}, {a: 1, b: "2", c: 3}]),
+        expectedOutput: EXPECTED_CSV_MULTIPLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: empty JSON",
+        input: JSON.stringify({}),
+        expectedOutput: EXPECTED_CSV_EMPTY,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: empty JSON in array",
+        input: JSON.stringify([{}]),
+        expectedOutput: EXPECTED_CSV_EMPTY,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    }
+]);