5.10.3

update

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "5.8.0",
+  "version": "5.10.3",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

src/core/Utils.js

@@ -259,6 +259,22 @@ const Utils = {
     },
 
 
+    /**
+     * Escape a string containing regex control characters so that it can be safely
+     * used in a regex without causing unintended behaviours.
+     *
+     * @param {string} str
+     * @returns {string}
+     *
+     * @example
+     * // returns "\[example\]"
+     * Utils.escapeRegex("[example]");
+     */
+    escapeRegex: function(str) {
+        return str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    },
+
+
     /**
      * Expand an alphabet range string into a list of the characters in that range.
      *
@@ -965,6 +981,37 @@ const Utils = {
     },
 
 
+    /**
+     * Parses URI parameters into a JSON object.
+     *
+     * @param {string} paramStr - The serialised query or hash section of a URI
+     * @returns {object}
+     *
+     * @example
+     * // returns {a: 'abc', b: '123'}
+     * Utils.parseURIParams("?a=abc&b=123")
+     * Utils.parseURIParams("#a=abc&b=123")
+     */
+    parseURIParams: function(paramStr) {
+        if (paramStr === "") return {};
+
+        // Cut off ? or # and split on &
+        const params = paramStr.substr(1).split("&");
+
+        const result = {};
+        for (let i = 0; i < params.length; i++) {
+            const param = params[i].split("=");
+            if (param.length !== 2) {
+                result[params[i]] = true;
+            } else {
+                result[param[0]] = decodeURIComponent(param[1].replace(/\+/g, " "));
+            }
+        }
+
+        return result;
+    },
+
+
     /**
      * Actual modulo function, since % is actually the remainder function in JS.
      *

src/core/config/Categories.js

@@ -126,6 +126,7 @@ const Categories = [
     {
         name: "Networking",
         ops: [
+            "HTTP request",
             "Strip HTTP headers",
             "Parse User Agent",
             "Parse IP range",
@@ -192,6 +193,8 @@ const Categories = [
             "Translate DateTime Format",
             "From UNIX Timestamp",
             "To UNIX Timestamp",
+            "Windows Filetime to UNIX Timestamp",
+            "UNIX Timestamp to Windows Filetime",
             "Extract dates",
         ]
     },

src/core/config/OperationConfig.js

@@ -1919,7 +1919,7 @@ const OperationConfig = {
         args: []
     },
     "Find / Replace": {
-        description: "Replaces all occurrences of the first string with the second.<br><br>The three match options are only relevant to regex search strings.",
+        description: "Replaces all occurrences of the first string with the second.<br><br> Includes support for regular expressions (regex), simple strings and extended strings (which support \\n, \\r, \\t, \\b, \\f and escaped hex bytes using \\x notation, e.g. \\x00 for a null byte).",
         run: StrUtils.runFindReplace,
         manualBake: true,
         inputType: "string",
@@ -2231,7 +2231,7 @@ const OperationConfig = {
         ]
     },
     "From UNIX Timestamp": {
-        description: "Converts a UNIX timestamp to a datetime string.<br><br>e.g. <code>978346800</code> becomes <code>Mon 1 January 2001 11:00:00 UTC</code>",
+        description: "Converts a UNIX timestamp to a datetime string.<br><br>e.g. <code>978346800</code> becomes <code>Mon 1 January 2001 11:00:00 UTC</code><br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).",
         run: DateTime.runFromUnixTimestamp,
         inputType: "number",
         outputType: "string",
@@ -2244,7 +2244,7 @@ const OperationConfig = {
         ]
     },
     "To UNIX Timestamp": {
-        description: "Parses a datetime string in UTC and returns the corresponding UNIX timestamp.<br><br>e.g. <code>Mon 1 January 2001 11:00:00</code> becomes <code>978346800</code>",
+        description: "Parses a datetime string in UTC and returns the corresponding UNIX timestamp.<br><br>e.g. <code>Mon 1 January 2001 11:00:00</code> becomes <code>978346800</code><br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).",
         run: DateTime.runToUnixTimestamp,
         inputType: "string",
         outputType: "number",
@@ -2261,6 +2261,32 @@ const OperationConfig = {
             }
         ]
     },
+    "Windows Filetime to UNIX Timestamp":{
+        description: "Converts a Windows Filetime value to a UNIX timestamp.<br><br>A Windows Filetime is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 UTC.<br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).<br><br>This operation also supports UNIX timestamps in milliseconds, microseconds and nanoseconds.",
+        run: DateTime.runFromFiletimeToUnix,
+        inputType: "string",
+        outputType: "string",
+        args: [
+            {
+                name: "Output units",
+                type: "option",
+                value: DateTime.UNITS
+            }
+        ]
+    },
+    "UNIX Timestamp to Windows Filetime":{
+        description: "Converts a UNIX timestamp to a Windows Filetime value.<br><br>A Windows Filetime is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 UTC.<br><br>A UNIX timestamp is a 32-bit value representing the number of seconds since January 1, 1970 UTC (the UNIX epoch).<br><br>This operation also supports UNIX timestamps in milliseconds, microseconds and nanoseconds.",
+        run: DateTime.runToFiletimeFromUnix,
+        inputType: "string",
+        outputType: "string",
+        args: [
+            {
+                name: "Input units",
+                type: "option",
+                value: DateTime.UNITS
+            }
+        ]
+    },
     "Translate DateTime Format": {
         description: "Parses a datetime string in one format and re-writes it in another.<br><br>Run with no input to see the relevant format string examples.",
         run: DateTime.runTranslateFormat,
@@ -3397,7 +3423,49 @@ const OperationConfig = {
         run: Image.runRemoveEXIF,
         inputType: "byteArray",
         outputType: "byteArray",
-        args: [],
+        args: []
+    },
+    "HTTP request": {
+        description: [
+            "Makes an HTTP request and returns the response.",
+            "<br><br>",
+            "This operation supports different HTTP verbs like GET, POST, PUT, etc.",
+            "<br><br>",
+            "You can add headers line by line in the format <code>Key: Value</code>",
+            "<br><br>",
+            "The status code of the response, along with a limited selection of exposed headers, can be viewed by checking the 'Show response metadata' option. Only a limited set of response headers are exposed by the browser for security reasons.",
+        ].join("\n"),
+        run: HTTP.runHTTPRequest,
+        inputType: "string",
+        outputType: "string",
+        manualBake: true,
+        args: [
+            {
+                name: "Method",
+                type: "option",
+                value: HTTP.METHODS,
+            },
+            {
+                name: "URL",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Headers",
+                type: "text",
+                value: "",
+            },
+            {
+                name: "Mode",
+                type: "option",
+                value: HTTP.MODE,
+            },
+            {
+                name: "Show response metadata",
+                type: "boolean",
+                value: false,
+            }
+        ]
     },
 };
 

src/core/operations/DateTime.js

@@ -1,3 +1,5 @@
+import {BigInteger} from "jsbn";
+
 /**
  * Date and time operations.
  *
@@ -78,6 +80,58 @@ const DateTime = {
     },
 
 
+    /**
+     * Windows Filetime to Unix Timestamp operation.
+     *
+     * @author bwhitn [brian.m.whitney@outlook.com]
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runFromFiletimeToUnix: function(input, args) {
+        let units = args[0];
+        input = new BigInteger(input).subtract(new BigInteger("116444736000000000"));
+        if (units === "Seconds (s)"){
+            input = input.divide(new BigInteger("10000000"));
+        } else if (units === "Milliseconds (ms)") {
+            input = input.divide(new BigInteger("10000"));
+        } else if (units === "Microseconds (μs)") {
+            input = input.divide(new BigInteger("10"));
+        } else if (units === "Nanoseconds (ns)") {
+            input = input.multiply(new BigInteger("100"));
+        } else {
+            throw "Unrecognised unit";
+        }
+        return input.toString();
+    },
+
+
+    /**
+     * Unix Timestamp to Windows Filetime operation.
+     *
+     * @author bwhitn [brian.m.whitney@outlook.com]
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runToFiletimeFromUnix: function(input, args) {
+        let units = args[0];
+        input = new BigInteger(input);
+        if (units === "Seconds (s)"){
+            input = input.multiply(new BigInteger("10000000"));
+        } else if (units === "Milliseconds (ms)") {
+            input = input.multiply(new BigInteger("10000"));
+        } else if (units === "Microseconds (μs)") {
+            input = input.multiply(new BigInteger("10"));
+        } else if (units === "Nanoseconds (ns)") {
+            input = input.divide(new BigInteger("100"));
+        } else {
+            throw "Unrecognised unit";
+        }
+        return input.add(new BigInteger("116444736000000000")).toString();
+    },
+
+
     /**
      * @constant
      * @default

src/core/operations/HTTP.js

@@ -12,6 +12,17 @@ import {UAS_parser as UAParser} from "../lib/uas_parser.js";
  */
 const HTTP = {
 
+    /**
+     * @constant
+     * @default
+     */
+    METHODS: [
+        "GET", "POST", "HEAD",
+        "PUT", "PATCH", "DELETE",
+        "CONNECT", "TRACE", "OPTIONS"
+    ],
+
+
     /**
      * Strip HTTP headers operation.
      *
@@ -51,6 +62,95 @@ const HTTP = {
             "Device Type: " + ua.deviceType + "\n";
     },
 
+
+    /**
+     * @constant
+     * @default
+     */
+    MODE: [
+        "Cross-Origin Resource Sharing",
+        "No CORS (limited to HEAD, GET or POST)",
+    ],
+
+    /**
+     * Lookup table for HTTP modes
+     *
+     * @private
+     * @constant
+     */
+    _modeLookup: {
+        "Cross-Origin Resource Sharing": "cors",
+        "No CORS (limited to HEAD, GET or POST)": "no-cors",
+    },
+
+    /**
+     * HTTP request operation.
+     *
+     * @author tlwr [toby@toby.codes]
+     * @author n1474335 [n1474335@gmail.com]
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    runHTTPRequest(input, args) {
+        const method = args[0],
+            url = args[1],
+            headersText = args[2],
+            mode = args[3],
+            showResponseMetadata = args[4];
+
+        if (url.length === 0) return "";
+
+        let headers = new Headers();
+        headersText.split(/\r?\n/).forEach(line => {
+            line = line.trim();
+
+            if (line.length === 0) return;
+
+            let split = line.split(":");
+            if (split.length !== 2) throw `Could not parse header in line: ${line}`;
+
+            headers.set(split[0].trim(), split[1].trim());
+        });
+
+        let config = {
+            method: method,
+            headers: headers,
+            mode: HTTP._modeLookup[mode],
+            cache: "no-cache",
+        };
+
+        if (method !== "GET" && method !== "HEAD") {
+            config.body = input;
+        }
+
+        return fetch(url, config)
+        .then(r => {
+            if (r.status === 0 && r.type === "opaque") {
+                return "Error: Null response. Try setting the connection mode to CORS.";
+            }
+
+            if (showResponseMetadata) {
+                let headers = "";
+                for (let pair of r.headers.entries()) {
+                    headers += "    " + pair[0] + ": " + pair[1] + "\n";
+                }
+                return r.text().then(b => {
+                    return "####\n  Status: " + r.status + " " + r.statusText +
+                        "\n  Exposed headers:\n" + headers + "####\n\n" + b;
+                });
+            }
+            return r.text();
+        })
+        .catch(e => {
+            return e.toString() +
+                "\n\nThis error could be caused by one of the following:\n" +
+                " - An invalid URL\n" +
+                " - Making a request to an insecure resource (HTTP) from a secure source (HTTPS)\n" +
+                " - Making a cross-origin request to a server which does not support CORS\n";
+        });
+    },
+
 };
 
 export default HTTP;

src/core/operations/StrUtils.js

@@ -227,14 +227,16 @@ const StrUtils = {
 
         if (type === "Regex") {
             find = new RegExp(find, modifiers);
-        } else if (type.indexOf("Extended") === 0) {
+            return input.replace(find, replace);
+        }
+
+        if (type.indexOf("Extended") === 0) {
             find = Utils.parseEscapedChars(find);
         }
 
-        return input.replace(find, replace, modifiers);
-        // Non-standard addition of flags in the third argument. This will work in Firefox but
-        // probably nowhere else. The purpose is to allow global matching when the `find` parameter
-        // is just a string.
+        find = new RegExp(Utils.escapeRegex(find), modifiers);
+
+        return input.replace(find, replace);
     },
 
 

src/web/App.js

@@ -21,21 +21,22 @@ import Split from "split.js";
  * @param {Object} options - Default setting for app options.
  */
 const App = function(categories, operations, defaultFavourites, defaultOptions) {
-    this.categories  = categories;
-    this.operations  = operations;
-    this.dfavourites = defaultFavourites;
-    this.doptions    = defaultOptions;
-    this.options     = Utils.extend({}, defaultOptions);
+    this.categories    = categories;
+    this.operations    = operations;
+    this.dfavourites   = defaultFavourites;
+    this.doptions      = defaultOptions;
+    this.options       = Utils.extend({}, defaultOptions);
 
-    this.chef        = new Chef();
-    this.manager     = new Manager(this);
+    this.chef          = new Chef();
+    this.manager       = new Manager(this);
 
-    this.baking      = false;
-    this.autoBake_   = false;
-    this.progress    = 0;
-    this.ingId       = 0;
+    this.baking        = false;
+    this.autoBake_     = false;
+    this.autoBakePause = false;
+    this.progress      = 0;
+    this.ingId         = 0;
 
-    window.chef      = this.chef;
+    window.chef        = this.chef;
 };
 
 
@@ -166,7 +167,7 @@ App.prototype.bake = async function(step) {
  * Runs Auto Bake if it is set.
  */
 App.prototype.autoBake = function() {
-    if (this.autoBake_) {
+    if (this.autoBake_ && !this.autoBakePause) {
         this.bake();
     }
 };
@@ -398,39 +399,28 @@ App.prototype.addFavourite = function(name) {
  * Checks for input and recipe in the URI parameters and loads them if present.
  */
 App.prototype.loadURIParams = function() {
-    // Load query string from URI
-    this.queryString = (function(a) {
-        if (a === "") return {};
-        const b = {};
-        for (let i = 0; i < a.length; i++) {
-            const p = a[i].split("=");
-            if (p.length !== 2) {
-                b[a[i]] = true;
-            } else {
-                b[p[0]] = decodeURIComponent(p[1].replace(/\+/g, " "));
-            }
-        }
-        return b;
-    })(window.location.search.substr(1).split("&"));
+    // Load query string or hash from URI (depending on which is populated)
+    const params = window.location.search || window.location.hash;
+    this.uriParams = Utils.parseURIParams(params);
 
-    // Turn off auto-bake while loading
-    const autoBakeVal = this.autoBake_;
-    this.autoBake_ = false;
+    // Pause auto-bake while loading but don't modify `this.autoBake_`
+    // otherwise `manualBake` cannot trigger.
+    this.autoBakePause = true;
 
-    // Read in recipe from query string
-    if (this.queryString.recipe) {
+    // Read in recipe from URI params
+    if (this.uriParams.recipe) {
         try {
-            const recipeConfig = JSON.parse(this.queryString.recipe);
+            const recipeConfig = JSON.parse(this.uriParams.recipe);
             this.setRecipeConfig(recipeConfig);
         } catch (err) {}
-    } else if (this.queryString.op) {
+    } else if (this.uriParams.op) {
         // If there's no recipe, look for single operations
         this.manager.recipe.clearRecipe();
         try {
-            this.manager.recipe.addOperation(this.queryString.op);
+            this.manager.recipe.addOperation(this.uriParams.op);
         } catch (err) {
             // If no exact match, search for nearest match and add that
-            const matchedOps = this.manager.ops.filterOperations(this.queryString.op, false);
+            const matchedOps = this.manager.ops.filterOperations(this.uriParams.op, false);
             if (matchedOps.length) {
                 this.manager.recipe.addOperation(matchedOps[0].name);
             }
@@ -438,21 +428,21 @@ App.prototype.loadURIParams = function() {
             // Populate search with the string
             const search = document.getElementById("search");
 
-            search.value = this.queryString.op;
+            search.value = this.uriParams.op;
             search.dispatchEvent(new Event("search"));
         }
     }
 
-    // Read in input data from query string
-    if (this.queryString.input) {
+    // Read in input data from URI params
+    if (this.uriParams.input) {
         try {
-            const inputData = Utils.fromBase64(this.queryString.input);
+            const inputData = Utils.fromBase64(this.uriParams.input);
             this.setInput(inputData);
         } catch (err) {}
     }
 
-    // Restore auto-bake state
-    this.autoBake_ = autoBakeVal;
+    // Unpause auto-bake
+    this.autoBakePause = false;
     this.autoBake();
 };
 

src/web/ControlsWaiter.js

@@ -174,20 +174,21 @@ ControlsWaiter.prototype.generateStateUrl = function(includeRecipe, includeInput
     const inputStr = Utils.toBase64(this.app.getInput(), "A-Za-z0-9+/"); // B64 alphabet with no padding
 
     includeRecipe = includeRecipe && (recipeConfig.length > 0);
-    includeInput = includeInput && (inputStr.length > 0) && (inputStr.length < 8000);
+    // Only inlcude input if it is less than 50KB (51200 * 4/3 as it is Base64 encoded)
+    includeInput = includeInput && (inputStr.length > 0) && (inputStr.length <= 68267);
 
     const params = [
         includeRecipe ? ["recipe", recipeStr] : undefined,
         includeInput ? ["input", inputStr] : undefined,
     ];
 
-    const query = params
+    const hash = params
        .filter(v => v)
        .map(([key, value]) => `${key}=${encodeURIComponent(value)}`)
        .join("&");
 
-    if (query) {
-        return `${link}?${query}`;
+    if (hash) {
+        return `${link}#${hash}`;
     }
 
     return link;

src/web/RecipeWaiter.js

@@ -354,8 +354,7 @@ RecipeWaiter.prototype.buildRecipeOperation = function(el) {
         el.classList.add("flow-control-op");
     }
 
-    // Disable auto-bake if this is a manual op - this should be moved to the 'operationadd'
-    // handler after event restructuring
+    // Disable auto-bake if this is a manual op
     if (op.manualBake && this.app.autoBake_) {
         this.manager.controls.setAutoBake(false);
         this.app.alert("Auto-Bake is disabled by default when using this operation.", "info", 5000);

src/web/html/index.html

@@ -35,7 +35,7 @@
             // Load theme before the preloader is shown
             document.querySelector(":root").className = JSON.parse(localStorage.getItem("options")).theme;
 
-            // Cycle loading messages
+            // Define loading messages
             const loadingMsgs = [
                 "Proving P = NP...",
                 "Computing 6 x 9...",
@@ -49,15 +49,28 @@
                 "Navigating neural network...",
                 "Importing machine learning..."
             ];
+
+            // Shuffle array using Durstenfeld algorithm
+            for (let i = loadingMsgs.length - 1; i > 0; --i) {
+                const j = Math.floor(Math.random() * (i + 1));
+                const temp = loadingMsgs[i];
+                loadingMsgs[i] = loadingMsgs[j];
+                loadingMsgs[j] = temp;
+            }
+
+            // Show next loading message then move it to the end of the array
             function changeLoadingMsg() {
+                const msg = loadingMsgs.shift();
                 try {
                     const el = document.getElementById("preloader-msg");
                     el.className = "loading"; // Causes CSS transition on first message
-                    el.innerHTML = loadingMsgs[Math.floor(Math.random()*loadingMsgs.length)];
-                } catch (err) {}
+                    el.innerHTML = msg;
+                } catch (err) {} // Ignore errors if DOM not yet ready
+                loadingMsgs.push(msg);
             }
+
             changeLoadingMsg();
-            window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random()*500) + 500);
+            window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 1000) + 1000);
         </script>
     </head>
     <body>

src/web/static/ga.html

@@ -9,7 +9,7 @@
 
   ga('create', 'UA-85682716-2', 'auto');
 
-  // Specifying location.pathname here overrides the default URL which would include arguments.
+  // Specifying location.pathname here overrides the default URL which could include arguments.
   // This method prevents Google Analytics from logging any recipe or input data in the URL.
   ga('send', 'pageview', location.pathname);
 

test/index.js

@@ -16,6 +16,7 @@ import "./tests/operations/ByteRepr.js";
 import "./tests/operations/CharEnc.js";
 import "./tests/operations/Code.js";
 import "./tests/operations/Compress.js";
+import "./tests/operations/DateTime.js";
 import "./tests/operations/FlowControl.js";
 import "./tests/operations/Image.js";
 import "./tests/operations/MorseCode.js";

test/tests/operations/DateTime.js

@@ -0,0 +1,34 @@
+/**
+ * DateTime tests.
+ *
+ * @author bwhitn [brian.m.whitney@outlook.com]
+ *
+ * @copyright Crown Copyright 2017
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister.js";
+
+TestRegister.addTests([
+    {
+        name: "Filetime to Unix",
+        input: "129207366395297693",
+        expectedOutput: "1276263039529769300",
+        recipeConfig: [
+            {
+                op: "Windows Filetime to UNIX Timestamp",
+                args: ["Nanoseconds (ns)"],
+            },
+        ],
+    },
+    {
+        name: "Unix to Filetime",
+        input: "1276263039529769300",
+        expectedOutput: "129207366395297693",
+        recipeConfig: [
+            {
+                op: "UNIX Timestamp to Windows Filetime",
+                args: ["Nanoseconds (ns)"],
+            },
+        ],
+    },
+]);