build: push tvapp v2 docker files

node_modules/user-agents/src/gunzip-data.js

@@ -0,0 +1,22 @@
+import fs from 'fs';
+import { gunzipSync } from 'zlib';
+
+
+const gunzipData = (inputFilename) => {
+  if (!inputFilename || !inputFilename.endsWith('.gz')) {
+    throw new Error('Filename must be specified and end with `.gz` for gunzipping.');
+  }
+  const outputFilename = inputFilename.slice(0, -3);
+  const compressedData = fs.readFileSync(inputFilename);
+  const data = gunzipSync(compressedData);
+  fs.writeFileSync(outputFilename, data);
+};
+
+
+if (!module.parent) {
+  const inputFilename = process.argv[2];
+  gunzipData(inputFilename);
+}
+
+
+export default gunzipData;

node_modules/user-agents/src/index.js

@@ -0,0 +1,4 @@
+import UserAgent from './user-agent';
+
+
+export default UserAgent;

node_modules/user-agents/src/update-data.js

@@ -0,0 +1,149 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import fs from "fs";
+import { gzipSync } from "zlib";
+
+import * as dynamoose from "dynamoose";
+import stableStringify from "fast-json-stable-stringify";
+import isbot from "isbot";
+import random from "random";
+import UAParser from "ua-parser-js";
+
+const ddb = new dynamoose.aws.ddb.DynamoDB({
+  region: "us-east-2",
+});
+dynamoose.aws.ddb.set(ddb);
+
+const SubmissionModel = dynamoose.model(
+  "userAgentsAnalyticsSubmissionTable",
+  new dynamoose.Schema(
+    {
+      id: {
+        type: String,
+        hashKey: true,
+      },
+      ip: String,
+      profile: Object,
+    },
+    {
+      saveUnknown: ["profile.**"],
+      timestamps: { createdAt: "timestamp", updatedAt: undefined },
+    },
+  ),
+  { create: false, update: false },
+);
+
+const getUserAgentTable = async (limit = 1e4) => {
+  const minimumTimestamp = Date.now() - 1 * 24 * 60 * 60 * 1000;
+
+  // Scan through all recent profiles keeping track of the count of each.
+  let lastKey = null;
+  const countsByProfile = {};
+  let totalCount = 0;
+  let uniqueCount = 0;
+  let ipAddressAlreadySeen = {};
+  do {
+    const scan = SubmissionModel.scan(
+      new dynamoose.Condition().filter("timestamp").gt(minimumTimestamp),
+    );
+    if (lastKey) {
+      scan.startAt(lastKey);
+    }
+
+    const response = await scan.exec();
+    response.forEach(({ ip, profile }) => {
+      // Only count one profile per IP address.
+      if (ipAddressAlreadySeen[ip]) return;
+      ipAddressAlreadySeen[ip] = true;
+
+      // Filter out bots like Googlebot and YandexBot.
+      if (isbot(profile.userAgent)) return;
+
+      // Track the counts for this exact profile.
+      const stringifiedProfile = stableStringify(profile);
+      if (!countsByProfile[stringifiedProfile]) {
+        countsByProfile[stringifiedProfile] = 0;
+        uniqueCount += 1;
+      }
+      countsByProfile[stringifiedProfile] += 1;
+      totalCount += 1;
+    });
+
+    lastKey = response.lastKey;
+  } while (lastKey);
+
+  // Add some noise to the counts/weights.
+  const n = () => random.normal();
+  Object.entries(countsByProfile).forEach(([stringifiedProfile, count]) => {
+    const unnormalizedWeight =
+      Array(2 * count)
+        .fill()
+        .reduce((sum) => sum + n()() ** 2, 0) / 2;
+    countsByProfile[stringifiedProfile] = unnormalizedWeight;
+  });
+
+  // Accumulate the profiles and add/remove a few properties to match the historical format.
+  const profiles = [];
+  for (let stringifiedProfile in countsByProfile) {
+    if (countsByProfile.hasOwnProperty(stringifiedProfile)) {
+      const profile = JSON.parse(stringifiedProfile);
+      profile.weight = countsByProfile[stringifiedProfile];
+      delete profile.sessionId;
+
+      // Deleting these because they weren't in the old format, but we should leave them in...
+      delete profile.language;
+      delete profile.oscpu;
+
+      // Find the device category.
+      const parser = new UAParser(profile.userAgent);
+      const device = parser.getDevice();
+      // Sketchy, but I validated this on historical data and it is a 100% match.
+      profile.deviceCategory =
+        { mobile: "mobile", tablet: "tablet", undefined: "desktop" }[
+          `${device.type}`
+        ] ?? "desktop";
+
+      profiles.push(profile);
+      delete countsByProfile[stringifiedProfile];
+    }
+  }
+
+  // Sort by descending weight.
+  profiles.sort((a, b) => b.weight - a.weight);
+
+  // Apply the count limit and normalize the weights.
+  profiles.splice(limit);
+  const totalWeight = profiles.reduce(
+    (total, profile) => total + profile.weight,
+    0,
+  );
+  profiles.forEach((profile) => {
+    profile.weight /= totalWeight;
+  });
+
+  return profiles;
+};
+
+if (!module.parent) {
+  const filename = process.argv[2];
+  if (!filename) {
+    throw new Error(
+      "An output filename must be passed as an argument to the command.",
+    );
+  }
+  getUserAgentTable()
+    .then(async (userAgents) => {
+      const stringifiedUserAgents = JSON.stringify(userAgents, null, 2);
+      // Compress the content if the extension ends with `.gz`.
+      const content = filename.endsWith(".gz")
+        ? gzipSync(stringifiedUserAgents)
+        : stringifiedUserAgents;
+      fs.writeFileSync(filename, content);
+    })
+    .catch((error) => {
+      // eslint-disable-next-line no-console
+      console.error(error);
+      process.exit(1);
+    });
+}
+
+export default getUserAgentTable;

node_modules/user-agents/src/user-agent.js

@@ -0,0 +1,154 @@
+import cloneDeep from 'lodash.clonedeep';
+
+import userAgents from './user-agents.json';
+
+
+// Normalizes the total weight to 1 and constructs a cumulative distribution.
+const makeCumulativeWeightIndexPairs = (weightIndexPairs) => {
+  const totalWeight = weightIndexPairs.reduce((sum, [weight]) => sum + weight, 0);
+  let sum = 0;
+  return weightIndexPairs.map(([weight, index]) => {
+    sum += weight / totalWeight;
+    return [sum, index];
+  });
+};
+
+// Precompute these so that we can quickly generate unfiltered user agents.
+const defaultWeightIndexPairs = userAgents.map(({ weight }, index) => [weight, index]);
+const defaultCumulativeWeightIndexPairs = makeCumulativeWeightIndexPairs(defaultWeightIndexPairs);
+
+
+// Turn the various filter formats into a single filter function that acts on raw user agents.
+const constructFilter = (filters, accessor = parentObject => parentObject) => {
+  let childFilters;
+  if (typeof filters === 'function') {
+    childFilters = [filters];
+  } else if (filters instanceof RegExp) {
+    childFilters = [
+      value => (
+        typeof value === 'object' && value && value.userAgent
+          ? filters.test(value.userAgent)
+          : filters.test(value)
+      ),
+    ];
+  } else if (filters instanceof Array) {
+    childFilters = filters.map(childFilter => constructFilter(childFilter));
+  } else if (typeof filters === 'object') {
+    childFilters = Object.entries(filters).map(([key, valueFilter]) => (
+      constructFilter(valueFilter, parentObject => parentObject[key])
+    ));
+  } else {
+    childFilters = [
+      value => (
+        typeof value === 'object' && value && value.userAgent
+          ? filters === value.userAgent
+          : filters === value
+      ),
+    ];
+  }
+
+  return (parentObject) => {
+    try {
+      const value = accessor(parentObject);
+      return childFilters.every(childFilter => childFilter(value));
+    } catch (error) {
+      // This happens when a user-agent lacks a nested property.
+      return false;
+    }
+  };
+};
+
+
+// Construct normalized cumulative weight index pairs given the filters.
+const constructCumulativeWeightIndexPairsFromFilters = (filters) => {
+  if (!filters) {
+    return defaultCumulativeWeightIndexPairs;
+  }
+
+  const filter = constructFilter(filters);
+
+  const weightIndexPairs = [];
+  userAgents.forEach((rawUserAgent, index) => {
+    if (filter(rawUserAgent)) {
+      weightIndexPairs.push([rawUserAgent.weight, index]);
+    }
+  });
+  return makeCumulativeWeightIndexPairs(weightIndexPairs);
+};
+
+
+const setCumulativeWeightIndexPairs = (userAgent, cumulativeWeightIndexPairs) => {
+  Object.defineProperty(userAgent, 'cumulativeWeightIndexPairs', {
+    configurable: true,
+    enumerable: false,
+    writable: false,
+    value: cumulativeWeightIndexPairs,
+  });
+};
+
+
+export default class UserAgent extends Function {
+  constructor(filters) {
+    super();
+    setCumulativeWeightIndexPairs(this, constructCumulativeWeightIndexPairsFromFilters(filters));
+    if (this.cumulativeWeightIndexPairs.length === 0) {
+      throw new Error('No user agents matched your filters.');
+    }
+
+    this.randomize();
+
+    return new Proxy(this, {
+      apply: () => this.random(),
+      get: (target, property, receiver) => {
+        const dataCandidate = target.data && typeof property === 'string'
+          && Object.prototype.hasOwnProperty.call(target.data, property)
+          && Object.prototype.propertyIsEnumerable.call(target.data, property);
+        if (dataCandidate) {
+          const value = target.data[property];
+          if (value !== undefined) {
+            return value;
+          }
+        }
+
+        return Reflect.get(target, property, receiver);
+      },
+    });
+  }
+
+  static random = (filters) => {
+    try {
+      return new UserAgent(filters);
+    } catch (error) {
+      return null;
+    }
+  };
+
+  //
+  // Standard Object Methods
+  //
+
+  [Symbol.toPrimitive] = () => (
+    this.data.userAgent
+  );
+
+  toString = () => (
+    this.data.userAgent
+  );
+
+  random = () => {
+    const userAgent = new UserAgent();
+    setCumulativeWeightIndexPairs(userAgent, this.cumulativeWeightIndexPairs);
+    userAgent.randomize();
+    return userAgent;
+  };
+
+  randomize = () => {
+    // Find a random raw random user agent.
+    const randomNumber = Math.random();
+    const [, index] = this.cumulativeWeightIndexPairs
+      .find(([cumulativeWeight]) => cumulativeWeight > randomNumber);
+    const rawUserAgent = userAgents[index];
+
+    this.data = cloneDeep(rawUserAgent);
+  }
+}