8.32.1

Merge branch 'oct' of https://github.com/wh0/CyberChef into wh0-oct
8.32.0
2019-06-27 15:56:16 +01:00 · 2019-06-27 15:55:34 +01:00 · 2019-06-27 15:50:30 +01:00 · 2019-06-27 15:45:17 +01:00 · 2019-06-27 15:42:32 +01:00 · 2019-06-27 15:41:20 +01:00
26 changed files with 1843 additions and 952 deletions
@@ -1,6 +1,6 @@
 language: node_js
 node_js:
-  - node
+  - lts/*
 addons:
  chrome: stable
 install: npm install
@@ -30,8 +30,9 @@ deploy:
    skip_cleanup: true
    api_key:
      secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA="
+    file_glob: true
    file:
-      - build/prod/cyberchef.htm
+      - build/prod/*.zip
      - build/node/CyberChef.js
    on:
      repo: gchq/CyberChef
@@ -2,6 +2,12 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).


+### [8.32.0] - 2019-06-27
+- 'Indec of Coincidence' operation added [@Ge0rg3] | [#571]
+
+### [8.31.0] - 2019-04-12
+- The downloadable version of CyberChef is now a .zip file containing separate modules rather than a single .htm file. It is still completely standalone and will not make any external network requests. This change reduces the complexity of the build process significantly. [@n1474335]
+
 ### [8.30.0] - 2019-04-12
 - 'Decode Protobuf' operation added [@n1474335] | [#533]

@@ -127,6 +133,8 @@ All major and minor version changes will be documented in this file. Details of



+[8.32.0]: https://github.com/gchq/CyberChef/releases/tag/v8.32.0
+[8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0
 [8.30.0]: https://github.com/gchq/CyberChef/releases/tag/v8.30.0
 [8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
 [8.28.0]: https://github.com/gchq/CyberChef/releases/tag/v8.28.0
@@ -185,6 +193,7 @@ All major and minor version changes will be documented in this file. Details of
 [@Cynser]: https://github.com/Cynser
 [@anthony-arnold]: https://github.com/anthony-arnold
 [@masq]: https://github.com/masq
+[@Ge0rg3]: https://github.com/Ge0rg3

 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -228,3 +237,4 @@ All major and minor version changes will be documented in this file. Details of
 [#516]: https://github.com/gchq/CyberChef/pull/516
 [#525]: https://github.com/gchq/CyberChef/pull/525
 [#533]: https://github.com/gchq/CyberChef/pull/533
+[#571]: https://github.com/gchq/CyberChef/pull/571
@@ -4,7 +4,6 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 const NodeExternals = require("webpack-node-externals");
-const Inliner = require("web-resource-inliner");
 const glob = require("glob");
 const path = require("path");

@@ -43,18 +42,16 @@ module.exports = function (grunt) {

    grunt.registerTask("prod",
        "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
+        [
+            "eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web",
+            "copy:standalone", "zip:standalone", "clean:standalone", "chmod"
+        ]);

    grunt.registerTask("default",
        "Lints the code base",
        ["eslint", "exec:repoSize"]);

-    grunt.registerTask("inline",
-        "Compiles a production build of CyberChef into a single, portable web page.",
-        ["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);

-
-    grunt.registerTask("runInliner", runInliner);
    grunt.registerTask("doc", "docs");
    grunt.registerTask("tests", "test");
    grunt.registerTask("lint", "eslint");
@@ -72,6 +69,7 @@ module.exports = function (grunt) {
    grunt.loadNpmTasks("grunt-accessibility");
    grunt.loadNpmTasks("grunt-concurrent");
    grunt.loadNpmTasks("grunt-contrib-connect");
+    grunt.loadNpmTasks("grunt-zip");


    // Project configuration
@@ -94,32 +92,6 @@ module.exports = function (grunt) {
        },
        moduleEntryPoints = listEntryModules();

-    /**
-     * Compiles a production build of CyberChef into a single, portable web page.
-     */
-    function runInliner() {
-        const done = this.async();
-        Inliner.html({
-            relativeTo: "build/prod/",
-            fileContent: grunt.file.read("build/prod/cyberchef.htm"),
-            images: true,
-            svgs: true,
-            scripts: true,
-            links: true,
-            strict: true
-        }, function(error, result) {
-            if (error) {
-                if (error instanceof Error) {
-                    done(error);
-                } else {
-                    done(new Error(error));
-                }
-            } else {
-                grunt.file.write("build/prod/cyberchef.htm", result);
-                done(true);
-            }
-        });
-    }

    /**
     * Generates an entry list for all the modules.
@@ -130,7 +102,7 @@ module.exports = function (grunt) {
        glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
            const basename = path.basename(file);
            if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
-                entryModules[basename.split(".mjs")[0]] = path.resolve(file);
+                entryModules["modules/" + basename.split(".mjs")[0]] = path.resolve(file);
        });

        return entryModules;
@@ -143,7 +115,7 @@ module.exports = function (grunt) {
            node: ["build/node/*"],
            config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
            docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
-            inlineScripts: ["build/prod/scripts.js"],
+            standalone: ["build/prod/CyberChef*.html"]
        },
        eslint: {
            options: {
@@ -195,6 +167,9 @@ module.exports = function (grunt) {
                    }, moduleEntryPoints),
                    output: {
                        path: __dirname + "/build/prod",
+                        filename: chunkData => {
+                            return chunkData.chunk.name === "main" ? "assets/[name].js": "[name].js";
+                        },
                        globalObject: "this"
                    },
                    resolve: {
@@ -225,33 +200,6 @@ module.exports = function (grunt) {
                    ]
                };
            },
-            webInline: {
-                mode: "production",
-                target: "web",
-                entry: "./src/web/index.js",
-                output: {
-                    filename: "scripts.js",
-                    path: __dirname + "/build/prod"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(Object.assign({}, BUILD_CONSTANTS, {
-                        INLINE: "true"
-                    })),
-                    new HtmlWebpackPlugin({
-                        filename: "cyberchef.htm",
-                        template: "./src/web/html/index.html",
-                        compileTime: compileTime,
-                        version: pkg.version + "s",
-                        inline: true,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
-                        }
-                    }),
-                ]
-            },
            node: {
                mode: "production",
                target: "node",
@@ -317,6 +265,18 @@ module.exports = function (grunt) {
                }
            }
        },
+        zip: {
+            standalone: {
+                cwd: "build/prod/",
+                src: [
+                    "build/prod/**/*",
+                    "!build/prod/index.html",
+                    "!build/prod/BundleAnalyzerReport.html",
+                    "!build/prod/sitemap.js"
+                ],
+                dest: `build/prod/CyberChef_v${pkg.version}.zip`
+            }
+        },
        connect: {
            prod: {
                options: {
@@ -329,10 +289,16 @@ module.exports = function (grunt) {
            ghPages: {
                options: {
                    process: function (content, srcpath) {
-                        // Add Google Analytics code to index.html
                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Add Google Analytics code to index.html
                            content = content.replace("</body></html>",
                                grunt.file.read("src/web/static/ga.html") + "</body></html>");
+
+                            // Add Structured Data for SEO
+                            content = content.replace("</head>",
+                                "<script type='application/ld+json'>" +
+                                JSON.stringify(JSON.parse(grunt.file.read("src/web/static/structuredData.json"))) +
+                                "</script></head>");
                            return grunt.template.process(content, srcpath);
                        } else {
                            return content;
@@ -351,6 +317,28 @@ module.exports = function (grunt) {
                        dest: "build/prod/"
                    },
                ]
+            },
+            standalone: {
+                options: {
+                    process: function (content, srcpath) {
+                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Replace download link with version number
+                            content = content.replace(/<a [^>]+>Download CyberChef.+?<\/a>/,
+                                `<span>Version ${pkg.version}</span>`);
+
+                            return grunt.template.process(content, srcpath);
+                        } else {
+                            return content;
+                        }
+                    },
+                    noProcess: ["**", "!**/*.html"]
+                },
+                files: [
+                    {
+                        src: "build/prod/index.html",
+                        dest: `build/prod/CyberChef_v${pkg.version}.html`
+                    }
+                ]
            }
        },
        chmod: {
@@ -406,7 +394,7 @@ module.exports = function (grunt) {
                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
            },
            browserTests: {
-                command: "./node_modules/.bin/nightwatch --env prod,inline"
+                command: "./node_modules/.bin/nightwatch --env prod"
            }
        },
    });
@@ -23,10 +23,6 @@

    "prod": {
      "launch_url": "http://localhost:8000/index.html"
-    },
-
-    "inline": {
-      "launch_url": "http://localhost:8000/cyberchef.htm"
    }

  }
@@ -1,6 +1,6 @@
 {
  "name": "cyberchef",
-  "version": "8.30.0",
+  "version": "8.32.1",
  "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
  "author": "n1474335 <n1474335@gmail.com>",
  "homepage": "https://gchq.github.io/CyberChef",
@@ -30,17 +30,17 @@
  "main": "build/node/CyberChef.js",
  "bugs": "https://github.com/gchq/CyberChef/issues",
  "devDependencies": {
-    "@babel/core": "^7.4.0",
-    "@babel/plugin-transform-runtime": "^7.4.0",
-    "@babel/preset-env": "^7.4.2",
-    "autoprefixer": "^9.5.0",
+    "@babel/core": "^7.4.4",
+    "@babel/plugin-transform-runtime": "^7.4.4",
+    "@babel/preset-env": "^7.4.4",
+    "autoprefixer": "^9.5.1",
    "babel-eslint": "^10.0.1",
    "babel-loader": "^8.0.5",
    "babel-plugin-syntax-dynamic-import": "^6.18.0",
-    "chromedriver": "^2.46.0",
+    "chromedriver": "^74.0.0",
    "colors": "^1.3.3",
    "css-loader": "^2.1.1",
-    "eslint": "^5.15.3",
+    "eslint": "^5.16.0",
    "exports-loader": "^0.7.0",
    "file-loader": "^3.0.1",
    "grunt": "^1.0.4",
@@ -53,34 +53,34 @@
    "grunt-contrib-watch": "^1.1.0",
    "grunt-eslint": "^21.0.0",
    "grunt-exec": "~3.0.0",
-    "grunt-jsdoc": "^2.3.0",
+    "grunt-jsdoc": "^2.4.0",
    "grunt-webpack": "^3.1.3",
+    "grunt-zip": "^0.18.2",
    "html-webpack-plugin": "^3.2.0",
    "imports-loader": "^0.8.0",
    "ink-docstrap": "^1.3.2",
    "jsdoc-babel": "^0.5.0",
-    "mini-css-extract-plugin": "^0.5.0",
+    "mini-css-extract-plugin": "^0.6.0",
    "nightwatch": "^1.0.19",
-    "node-sass": "^4.11.0",
+    "node-sass": "^4.12.0",
    "postcss-css-variables": "^0.12.0",
    "postcss-import": "^12.0.1",
    "postcss-loader": "^3.0.0",
    "prompt": "^1.0.0",
    "sass-loader": "^7.1.0",
-    "sitemap": "^2.1.0",
+    "sitemap": "^2.2.0",
    "style-loader": "^0.23.1",
    "svg-url-loader": "^2.3.2",
    "url-loader": "^1.1.2",
-    "web-resource-inliner": "^4.3.1",
-    "webpack": "^4.29.6",
-    "webpack-bundle-analyzer": "^3.1.0",
-    "webpack-dev-server": "^3.2.1",
+    "webpack": "^4.31.0",
+    "webpack-bundle-analyzer": "^3.3.2",
+    "webpack-dev-server": "^3.3.1",
    "webpack-node-externals": "^1.7.2",
    "worker-loader": "^2.0.0"
  },
  "dependencies": {
-    "@babel/polyfill": "^7.4.0",
-    "@babel/runtime": "^7.4.2",
+    "@babel/polyfill": "^7.4.4",
+    "@babel/runtime": "^7.4.4",
    "arrive": "^2.4.1",
    "babel-plugin-transform-builtin-extend": "1.1.2",
    "bcryptjs": "^2.4.3",
@@ -92,11 +92,11 @@
    "bson": "^4.0.2",
    "chi-squared": "^1.1.0",
    "clippyjs": "0.0.3",
-    "core-js": "^3.0.0",
+    "core-js": "^3.0.1",
    "crypto-api": "^0.8.3",
    "crypto-js": "^3.1.9-1",
    "ctph.js": "0.0.5",
-    "d3": "^4.9.1",
+    "d3": "^5.9.2",
    "d3-hexbin": "^0.2.2",
    "diff": "^4.0.1",
    "es6-promisify": "^6.0.1",
@@ -107,8 +107,8 @@
    "file-saver": "^2.0.1",
    "geodesy": "^1.1.3",
    "highlight.js": "^9.15.6",
-    "jimp": "^0.6.0",
-    "jquery": "^3.3.1",
+    "jimp": "^0.6.4",
+    "jquery": "3.4.1",
    "js-crc": "^0.2.0",
    "js-sha3": "^0.8.0",
    "jsesc": "^2.5.2",
@@ -122,7 +122,7 @@
    "loglevel": "^1.6.1",
    "loglevel-message-prefix": "^3.0.0",
    "moment": "^2.24.0",
-    "moment-timezone": "^0.5.23",
+    "moment-timezone": "^0.5.25",
    "ngeohash": "^0.6.3",
    "node-forge": "^0.8.2",
    "node-md6": "^0.1.0",
@@ -130,11 +130,11 @@
    "notepack.io": "^2.2.0",
    "nwmatcher": "^1.4.4",
    "otp": "^0.1.3",
-    "popper.js": "^1.14.7",
+    "popper.js": "^1.15.0",
    "qr-image": "^3.2.0",
    "scryptsy": "^2.0.0",
    "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.8.4",
+    "sortablejs": "^1.9.0",
    "split.js": "^1.5.10",
    "ssdeep.js": "0.0.2",
    "ua-parser-js": "^0.7.19",
@@ -178,7 +178,7 @@ self.loadRequiredModules = function(recipeConfig) {
        if (!OpModules.hasOwnProperty(module)) {
            log.info(`Loading ${module} module`);
            self.sendStatusMessage(`Loading ${module} module`);
-            self.importScripts(`${self.docURL}/${module}.js`);
+            self.importScripts(`${self.docURL}/modules/${module}.js`);
            self.sendStatusMessage("");
        }
    });
@@ -21,8 +21,8 @@ class Dish {
     * @param {Dish} [dish=null] - A dish to clone
     */
    constructor(dish=null) {
-        this.value = [];
-        this.type = Dish.BYTE_ARRAY;
+        this.value = new ArrayBuffer(0);
+        this.type = Dish.ARRAY_BUFFER;

        if (dish &&
            dish.hasOwnProperty("value") &&
@@ -201,11 +201,18 @@ class Utils {
     * Utils.parseEscapedChars("\\n");
     */
    static parseEscapedChars(str) {
-        return str.replace(/(\\)?\\([bfnrtv0'"]|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) {
+        return str.replace(/(\\)?\\([bfnrtv'"]|[0-3][0-7]{2}|[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) {
            if (a === "\\") return "\\"+b;
            switch (b[0]) {
                case "0":
-                    return "\0";
+                case "1":
+                case "2":
+                case "3":
+                case "4":
+                case "5":
+                case "6":
+                case "7":
+                    return String.fromCharCode(parseInt(b, 8));
                case "b":
                    return "\b";
                case "t":
@@ -836,7 +843,7 @@ class Utils {
            args = m[2]
                .replace(/"/g, '\\"') // Escape double quotes
                .replace(/(^|,|{|:)'/g, '$1"') // Replace opening ' with "
-                .replace(/([^\\]|[^\\]\\\\)'(,|:|}|$)/g, '$1"$2') // Replace closing ' with "
+                .replace(/([^\\]|(?:\\\\)+)'(,|:|}|$)/g, '$1"$2') // Replace closing ' with "
                .replace(/\\'/g, "'"); // Unescape single quotes
            args = "[" + args + "]";

@@ -395,6 +395,7 @@
        "ops": [
            "Entropy",
            "Frequency distribution",
+            "Index of Coincidence",
            "Chi Square",
            "Disassemble x86",
            "Pseudo-Random Number Generator",
@@ -65,8 +65,8 @@ class AESEncrypt extends Operation {
     * @throws {OperationError} if invalid key length
     */
    run(input, args) {
-        const key = Utils.convertToByteArray(args[0].string, args[0].option),
-            iv = Utils.convertToByteArray(args[1].string, args[1].option),
+        const key = Utils.convertToByteString(args[0].string, args[0].option),
+            iv = Utils.convertToByteString(args[1].string, args[1].option),
            mode = args[2],
            inputType = args[3],
            outputType = args[4];
@@ -0,0 +1,107 @@
+/**
+ * @author George O [georgeomnet+cyberchef@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+
+/**
+ * Index of Coincidence operation
+ */
+class IndexOfCoincidence extends Operation {
+
+    /**
+     * IndexOfCoincidence constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Index of Coincidence";
+        this.module = "Default";
+        this.description = "Index of Coincidence (IC) is the probability of two randomly selected characters being the same. This can be used to determine whether text is readable or random, with English text having an IC of around 0.066. IC can therefore be a sound method to automate frequency analysis.";
+        this.infoURL = "https://wikipedia.org/wiki/Index_of_coincidence";
+        this.inputType = "string";
+        this.outputType = "number";
+        this.presentType = "html";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {number}
+     */
+    run(input, args) {
+        const text = input.toLowerCase().replace(/[^a-z]/g, ""),
+            frequencies = new Array(26).fill(0),
+            alphabet = Utils.expandAlphRange("a-z");
+        let coincidence = 0.00,
+            density = 0.00,
+            result = 0.00,
+            i;
+
+        for (i=0; i < alphabet.length; i++) {
+            frequencies[i] = text.count(alphabet[i]);
+        }
+
+        for (i=0; i < frequencies.length; i++) {
+            coincidence += frequencies[i] * (frequencies[i] - 1);
+        }
+
+        density = frequencies.sum();
+
+        // Ensure that we don't divide by 0
+        if (density < 2) density = 2;
+
+        result = coincidence / (density * (density - 1));
+
+        return result;
+    }
+
+    /**
+     * Displays the IC as a scale bar for web apps.
+     *
+     * @param {number} ic
+     * @returns {html}
+     */
+    present(ic) {
+        return `Index of Coincidence: ${ic}
+Normalized: ${ic * 26}
+<br><canvas id='chart-area'></canvas><br>
+- 0 represents complete randomness (all characters are unique), whereas 1 represents no randomness (all characters are identical).
+- English text generally has an IC of between 0.67 to 0.78.
+- 'Random' text is determined by the probability that each letter occurs the same number of times as another.
+
+The graph shows the IC of the input data. A low IC generally means that the text is random, compressed or encrypted.
+
+<script type='application/javascript'>
+  var canvas = document.getElementById("chart-area"),
+      parentRect = canvas.parentNode.getBoundingClientRect(),
+      ic = ${ic};
+
+  canvas.width = parentRect.width * 0.95;
+  canvas.height = parentRect.height * 0.25;
+
+  ic = ic > 0.25 ? 0.25 : ic;
+
+  CanvasComponents.drawScaleBar(canvas, ic, 0.25, [
+    {
+      label: "English text",
+      min: 0.05,
+      max: 0.08
+    },
+    {
+      label: "> 0.25",
+      min: 0.24,
+      max: 0.25
+    }
+  ]);
+</script>
+     `;
+    }
+
+}
+
+export default IndexOfCoincidence;
@@ -51,6 +51,10 @@ class JSONToCSV extends Operation {
        this.rowDelim = rowDelim;
        const self = this;

+        if (!(input instanceof Array)) {
+            input = [input];
+        }
+
        try {
            // If the JSON is an array of arrays, this is easy
            if (input[0] instanceof Array) {
@@ -89,6 +93,8 @@ class JSONToCSV extends Operation {
     * @returns {string}
     */
    escapeCellContents(data) {
+        if (typeof data === "number") data = data.toString();
+
        // Double quotes should be doubled up
        data = data.replace(/"/g, '""');

@@ -230,6 +230,7 @@ function regexHighlight (input, regex, displayTotal) {
        title = "",
        hl = 1,
        total = 0;
+    const captureGroups = [];

    output = input.replace(regex, (match, ...args) => {
        args.pop(); // Throw away full string
@@ -247,9 +248,15 @@ function regexHighlight (input, regex, displayTotal) {
        // Switch highlight
        hl = hl === 1 ? 2 : 1;

-        total++;
+        // Store highlighted match and replace with a placeholder
+        captureGroups.push(`<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`);
+        return `[cc_capture_group_${total++}]`;
+    });

-        return `<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`;
+    // Safely escape all remaining text, then replace placeholders
+    output = Utils.escapeHtml(output);
+    output = output.replace(/\[cc_capture_group_(\d+)\]/g, (_, i) => {
+        return captureGroups[i];
    });

    if (displayTotal)
@@ -79,7 +79,7 @@ class TextEncodingBruteForce extends Operation {
        let table = "<table class='table table-hover table-sm table-bordered table-nonfluid'><tr><th>Encoding</th><th>Value</th></tr>";

        for (const enc in encodings) {
-            const value = Utils.printable(encodings[enc], true);
+            const value = Utils.escapeHtml(Utils.printable(encodings[enc], true));
            table += `<tr><td>${enc}</td><td>${value}</td></tr>`;
        }

@@ -20,7 +20,7 @@ class UnescapeString extends Operation {

        this.name = "Unescape string";
        this.module = "Default";
-        this.description = "Unescapes characters in a string that have been escaped. For example, <code>Don\\'t stop me now</code> becomes <code>Don't stop me now</code>.<br><br>Supports the following escape sequences:<ul><li><code>\\n</code> (Line feed/newline)</li><li><code>\\r</code> (Carriage return)</li><li><code>\\t</code> (Horizontal tab)</li><li><code>\\b</code> (Backspace)</li><li><code>\\f</code> (Form feed)</li><li><code>\\xnn</code> (Hex, where n is 0-f)</li><li><code>\\\\</code> (Backslash)</li><li><code>\\'</code> (Single quote)</li><li><code>\\&quot;</code> (Double quote)</li><li><code>\\unnnn</code> (Unicode character)</li><li><code>\\u{nnnnnn}</code> (Unicode code point)</li></ul>";
+        this.description = "Unescapes characters in a string that have been escaped. For example, <code>Don\\'t stop me now</code> becomes <code>Don't stop me now</code>.<br><br>Supports the following escape sequences:<ul><li><code>\\n</code> (Line feed/newline)</li><li><code>\\r</code> (Carriage return)</li><li><code>\\t</code> (Horizontal tab)</li><li><code>\\b</code> (Backspace)</li><li><code>\\f</code> (Form feed)</li><li><code>\\nnn</code> (Octal, where n is 0-7)</li><li><code>\\xnn</code> (Hex, where n is 0-f)</li><li><code>\\\\</code> (Backslash)</li><li><code>\\'</code> (Single quote)</li><li><code>\\&quot;</code> (Double quote)</li><li><code>\\unnnn</code> (Unicode character)</li><li><code>\\u{nnnnnn}</code> (Unicode code point)</li></ul>";
        this.infoURL = "https://wikipedia.org/wiki/Escape_sequence";
        this.inputType = "string";
        this.outputType = "string";
@@ -108,7 +108,7 @@ class App {
    handleError(err, logToConsole) {
        if (logToConsole) log.error(err);
        const msg = err.displayStr || err.toString();
-        this.alert(msg, this.options.errorTimeout, !this.options.showErrors);
+        this.alert(Utils.escapeHtml(msg), this.options.errorTimeout, !this.options.showErrors);
    }


@@ -338,7 +338,7 @@ class ControlsWaiter {
        const saveLink = this.generateStateUrl(true, true, null, "https://gchq.github.io/CyberChef/");

        if (reportBugInfo) {
-            reportBugInfo.innerHTML = `* Version: ${PKG_VERSION + (typeof INLINE === "undefined" ? "" : "s")}
+            reportBugInfo.innerHTML = `* Version: ${PKG_VERSION}
 * Compile time: ${COMPILE_TIME}
 * User-Agent:
 ${navigator.userAgent}
@@ -293,7 +293,9 @@ class HTMLIngredient {
        const op = el.parentNode.parentNode;
        const target = op.querySelectorAll(".arg")[this.target];

-        target.value = el.childNodes[el.selectedIndex].getAttribute("populate-value");
+        const popVal = el.childNodes[el.selectedIndex].getAttribute("populate-value");
+        if (popVal !== "") target.value = popVal;
+
        const evt = new Event("change");
        target.dispatchEvent(evt);

@@ -124,16 +124,21 @@ class RecipeWaiter {
     * @param {event} evt
     */
    opSortEnd(evt) {
-        if (this.removeIntent) {
-            if (evt.item.parentNode.id === "rec-list") {
-                evt.item.remove();
-            }
+        if (this.removeIntent && evt.item.parentNode.id === "rec-list") {
+            evt.item.remove();
            return;
        }

        // Reinitialise the popover on the original element in the ops list because for some reason it
-        // gets destroyed and recreated.
-        this.manager.ops.enableOpsListPopovers(evt.clone);
+        // gets destroyed and recreated. If the clone isn't in the ops list, we use the original item instead.
+        let enableOpsElement;
+        if (evt.clone.parentNode && evt.clone.parentNode.classList.contains("op-list")) {
+            enableOpsElement = evt.clone;
+        } else {
+            enableOpsElement = evt.item;
+            $(evt.item).attr("data-toggle", "popover");
+        }
+        this.manager.ops.enableOpsListPopovers(enableOpsElement);

        if (evt.item.parentNode.id !== "rec-list") {
            return;
@@ -131,13 +131,6 @@
            };
            window.addEventListener("error", loadingErrorHandler);
        </script>
-        <% if (htmlWebpackPlugin.options.inline) { %>
-            <meta name="robots" content="noindex" />
-        <% } else { %>
-            <script type="application/ld+json">
-                <% print(JSON.stringify(require("../static/structuredData.json"))); %>
-            </script>
-        <% } %>
    </head>
    <body>
        <!-- Preloader overlay -->
@@ -153,11 +146,7 @@
        <div id="content-wrapper">
            <div id="banner" class="row">
                <div class="col" style="text-align: left; padding-left: 10px;">
-                    <% if (htmlWebpackPlugin.options.inline) { %>
-                        <span>Version <%= htmlWebpackPlugin.options.version %></span>
-                    <% } else { %>
-                        <a href="cyberchef.htm" download>Download CyberChef <i class="material-icons">file_download</i></a>
-                    <% } %>
+                    <a href="CyberChef_v<%= htmlWebpackPlugin.options.version %>.zip" download>Download CyberChef <i class="material-icons">file_download</i></a>
                </div>
                <div class="col-md-6" id="notice-wrapper">
                    <span id="notice">
@@ -49,9 +49,11 @@ import "./tests/Hash";
 import "./tests/HaversineDistance";
 import "./tests/Hexdump";
 import "./tests/Image";
+import "./tests/IndexOfCoincidence";
 import "./tests/Jump";
 import "./tests/JSONBeautify";
 import "./tests/JSONMinify";
+import "./tests/JSONtoCSV";
 import "./tests/JWTDecode";
 import "./tests/JWTSign";
 import "./tests/JWTVerify";
@@ -209,9 +209,9 @@ Tag: 16a3e732a605cc9ca29108f742ca0743`,
    {
        name: "AES Encrypt: AES-128-GCM, Binary",
        input: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018",
-        expectedOutput: `fa17fcbf5e8763322c1b0c8562e1512ed9d702ef70c1643572b9de3e34ae6b535e6c1b992432aa6d06fb6f80c861262aef66e7c26035afe77bd3861261e4e092b523f058f8ebef2143db21bc16d02f7a011efb07419300cb41c3b884d1d8d6a766b8963c
+        expectedOutput: `5a29debb5c5f38cdf8aee421bd94dbbf3399947faddf205f88b3ad8ecb0c51214ec0e28bf78942dfa212d7eb15259bbdcac677b4c05f473eeb9331d74f31d441d97d56eb5c73b586342d72128ca528813543dc0fc7eddb7477172cc9194c18b2e1383e4e

-Tag: fa6bbb34c8cde65a3d7b93fb094fc84f`,
+Tag: 70fad2ca19412c20f40fd06918736e56`,
        recipeConfig: [
            {
                "op": "AES Encrypt",
@@ -301,9 +301,9 @@ Tag: fa6bbb34c8cde65a3d7b93fb094fc84f`,
    {
        name: "AES Encrypt: AES-192-GCM, Binary",
        input: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018",
-        expectedOutput: `ed22946f96964d300b45f5ce2d9601ba87682da1a603c90e6d4f7738729b0602f613ee392c9bfc7792594474f1213fb99185851f02ece4df0e93995e49f97aa4d0a337d7a80d83e4219dae5a3d36658f8659cdd5ed7c32707f98656fab7fb43f7a61e37c
+        expectedOutput: `318b479d919d506f0cd904f2676fab263a7921b6d7e0514f36e03ae2333b77fa66ef5600babcb2ee9718aeb71fc357412343c1f2cb351d8715bb0aedae4a6468124f9c4aaf6a721b306beddbe63a978bec8baeeba4b663be33ee5bc982746bd4aed1c38b

-Tag: be17cb31edb77f648b9d1032b235b33d`,
+Tag: 86db597d5302595223cadbd990f1309b`,
        recipeConfig: [
            {
                "op": "AES Encrypt",
@@ -393,9 +393,9 @@ Tag: be17cb31edb77f648b9d1032b235b33d`,
    {
        name: "AES Encrypt: AES-256-GCM, Binary",
        input: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018",
-        expectedOutput: `e3f1b236eaf3b9df69df8133a1b417fa42b242d8ad49e4d2f3469aca7e2a41737e4f2c8a0d212143287088fad51743577dc6dfa8ed328ca90113cbeb9b137926b2168cc037bdc371777e6ee02b9d9c017b6054fd83d43b4885fbe9c044a8574f1491a893
+        expectedOutput: `1287f188ad4d7ab0d9ff69b3c29cb11f861389532d8cb9337181da2e8cfc74a84927e8c0dd7a28a32fd485afe694259a63c199b199b95edd87c7aa95329feac340f2b78b72956a85f367044d821766b1b7135815571df44900695f1518cf3ae38ecb650f

-Tag: 23ddbd3ee4de33f98a9ea9a170bdf268`,
+Tag: 821b1e5f32dad052e502775a523d957a`,
        recipeConfig: [
            {
                "op": "AES Encrypt",
@@ -0,0 +1,22 @@
+/**
+ * Index of Coincidence tests.
+ *
+ * @author George O [georgeomnet+cyberchef@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Index of Coincidence",
+        input: "Hello world, this is a test to determine the correct IC value.",
+        expectedMatch: /^Index of Coincidence: 0\.07142857142857142\nNormalized: 1\.857142857142857/,
+        recipeConfig: [
+            {
+                "op": "Index of Coincidence",
+                "args": []
+            },
+        ],
+    },
+]);
@@ -0,0 +1,93 @@
+/**
+ * JSON to CSV tests.
+ *
+ * @author mshwed [m@ttshwed.com]
+ *
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+const EXPECTED_CSV_SINGLE = "a,b,c\r\n1,2,3\r\n";
+const EXPECTED_CSV_MULTIPLE = "a,b,c\r\n1,2,3\r\n1,2,3\r\n";
+const EXPECTED_CSV_EMPTY = "\r\n\r\n";
+
+TestRegister.addTests([
+    {
+        name: "JSON to CSV: strings as values",
+        input: JSON.stringify({a: "1", b: "2", c: "3"}),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: numbers as values",
+        input: JSON.stringify({a: 1, b: 2, c: 3}),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: numbers and strings as values",
+        input: JSON.stringify({a: 1, b: "2", c: 3}),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: JSON as an array",
+        input: JSON.stringify([{a: 1, b: "2", c: 3}]),
+        expectedOutput: EXPECTED_CSV_SINGLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: multiple JSON values in an array",
+        input: JSON.stringify([{a: 1, b: "2", c: 3}, {a: 1, b: "2", c: 3}]),
+        expectedOutput: EXPECTED_CSV_MULTIPLE,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: empty JSON",
+        input: JSON.stringify({}),
+        expectedOutput: EXPECTED_CSV_EMPTY,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    },
+    {
+        name: "JSON to CSV: empty JSON in array",
+        input: JSON.stringify([{}]),
+        expectedOutput: EXPECTED_CSV_EMPTY,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\\r\\n"]
+            },
+        ],
+    }
+]);
@@ -48,7 +48,7 @@ module.exports = {
            "process.browser": "true"
        }),
        new MiniCssExtractPlugin({
-            filename: "[name].css"
+            filename: "assets/[name].css"
        }),
    ],
    resolve: {
@@ -80,7 +80,12 @@ module.exports = {
            {
                test: /\.css$/,
                use: [
-                    MiniCssExtractPlugin.loader,
+                    {
+                        loader: MiniCssExtractPlugin.loader,
+                        options: {
+                            publicPath: "../"
+                        }
+                    },
                    "css-loader",
                    "postcss-loader",
                ]
@@ -88,7 +93,12 @@ module.exports = {
            {
                test: /\.scss$/,
                use: [
-                    MiniCssExtractPlugin.loader,
+                    {
+                        loader: MiniCssExtractPlugin.loader,
+                        options: {
+                            publicPath: "../"
+                        }
+                    },
                    "css-loader",
                    "sass-loader",
                ]
@@ -97,7 +107,9 @@ module.exports = {
                test: /\.(ico|eot|ttf|woff|woff2)$/,
                loader: "url-loader",
                options: {
-                    limit: 10000
+                    limit: 10000,
+                    name: "[hash].[ext]",
+                    outputPath: "assets"
                }
            },
            {
@@ -120,7 +132,9 @@ module.exports = {
                exclude: /web\/static/,
                loader: "url-loader",
                options: {
-                    limit: 10000
+                    limit: 10000,
+                    name: "[hash].[ext]",
+                    outputPath: "assets"
                }
            },
        ]
Author	SHA1	Message	Date
n1474335	c0983654d8	8.32.1	2019-06-27 15:56:16 +01:00
n1474335	30bf95f2c1	Merge branch 'oct' of https://github.com/wh0/CyberChef into wh0-oct	2019-06-27 15:55:34 +01:00
n1474335	da178107f9	8.32.0	2019-06-27 15:50:30 +01:00
n1474335	4dda1d9e49	Updated CHANGELOG	2019-06-27 15:45:17 +01:00
n1474335	e11aec64cd	Modified wording for IC op	2019-06-27 15:42:32 +01:00
n1474335	71575e49d7	Merge branch 'index-of-coincidence' of https://github.com/Ge0rg3/CyberChef into Ge0rg3-index-of-coincidence	2019-06-27 15:41:20 +01:00
n1474335	393d070b05	8.31.8	2019-06-27 15:37:12 +01:00
n1474335	d7e4c28cd0	Merge branch 'bugfix/json-to-csv' of https://github.com/MShwed/CyberChef into MShwed-bugfix/json-to-csv	2019-06-27 15:36:12 +01:00
n1474335	ccf2348cd6	8.31.7	2019-06-27 15:34:22 +01:00
n1474335	cde3eb2c39	Merge branch 'popover-patch' of https://github.com/Ge0rg3/CyberChef into Ge0rg3-popover-patch	2019-06-27 15:31:57 +01:00
wh0	f29d8eeda8	Parse octal escape sequences	2019-06-25 18:35:25 -07:00
mshwed	313d1a580e	Merge branch 'master' of https://github.com/gchq/CyberChef into bugfix/json-to-csv	2019-06-20 13:59:08 -04:00
j433866	ca9bab5d4c	8.31.6	2019-06-14 14:32:15 +01:00
j433866	e35ef8f39b	Escape HTML for error messages being sent to alert	2019-06-14 14:31:38 +01:00
George O	e709582062	Disappearing Popover Fix	2019-06-10 19:19:50 +01:00
George O	a6732ba815	Added Index of Coincidence Tests	2019-06-09 00:57:02 +01:00
George O	466d872d30	Added Index of Coincidence Operation	2019-06-09 00:57:02 +01:00
j433866	3cf7238106	8.31.5	2019-06-07 11:04:00 +01:00
j433866	f8d08cc5db	Merge branch 'j433866-aes-gcm-fix'	2019-06-07 10:55:53 +01:00
j433866	c1bdca8df3	Change to use byte strings instead of byte array	2019-06-04 10:19:00 +01:00
j433866	2be2c83f67	8.31.4	2019-05-21 16:56:23 +01:00
j433866	a271eaabd0	Merge branch 'j433866-load-regex-fix'	2019-05-21 16:53:47 +01:00
j433866	6f5018d45e	Don't change option value if new value is blank	2019-05-16 11:59:25 +01:00
j433866	f51ee76c72	Merge pull request #3 from gchq/master Bring up to date with gchq/master	2019-05-16 11:53:58 +01:00
j433866	0c9e8fe050	8.31.3	2019-05-09 17:07:12 +01:00
j433866	8f41571e47	Merge branch 'd98762625-use-node-lts'	2019-05-09 16:43:08 +01:00
j433866	6d14368e2f	Merge branch 'use-node-lts' of https://github.com/d98762625/CyberChef into d98762625-use-node-lts	2019-05-09 16:42:19 +01:00
j433866	f90ad48906	Update dependencies	2019-05-09 16:20:42 +01:00
j433866	e95dac82c2	8.31.2	2019-05-09 11:56:27 +01:00
j433866	144601ffd4	Merge branch 'j433866-load-recipe-fix'	2019-05-09 11:54:18 +01:00
j433866	cbcc2aa731	Fix regex to handle multiple escaped backslashes	2019-05-09 10:04:06 +01:00
j433866	f9354c8cd1	Merge remote-tracking branch 'upstream/master'	2019-05-09 09:08:01 +01:00
d98762625	7a4f418e75	configure travis to use node LTS instead of latest	2019-04-29 17:10:40 +01:00
mshwed	8fa8e34027	Added support for parsing JSON with number type values. Added support for non-array JSON objects. Added extra tests for JSON to CSV operation.	2019-04-28 16:29:15 -04:00
n1474335	01f0625d6a	Fixed XSS in 'Text Encoding Brute Force. Closes #539	2019-04-14 22:00:17 +01:00
n1474335	38ff7ec89f	8.31.1	2019-04-14 21:56:01 +01:00
n1474335	7163a0802d	Tidied up build directory	2019-04-14 21:55:52 +01:00
n1474335	8d0fcf37c5	8.31.0	2019-04-12 18:58:48 +01:00
n1474335	3da5a8bb34	Merge branch 'downloadable-zip'	2019-04-12 18:58:36 +01:00
n1474335	fad33b583b	Updated CHANGELOG	2019-04-12 18:58:22 +01:00
n1474335	8f450501cc	Downloadble version is now a .zip file instead of a single .htm file	2019-04-12 18:54:31 +01:00
n1474335	b3ae0e577a	8.30.1	2019-04-12 16:25:47 +01:00
n1474335	aedac94e40	Merge branch 'd98762625-init-dish'	2019-04-12 16:25:38 +01:00
n1474335	08c5dbce09	Merge branch 'init-dish' of https://github.com/d98762625/CyberChef into d98762625-init-dish	2019-04-12 16:24:23 +01:00
d98762625	cdc15c0f20	change empty Dish initialisation to use ArrayBuffer	2019-04-08 17:58:46 +01:00
j433866	328c0ade22	Merge remote-tracking branch 'upstream/master'	2019-03-19 14:21:28 +00:00
j433866	0a7a0ac681	Merge pull request #1 from gchq/master bring up to date with master	2019-03-12 13:27:24 +00:00