Research-Stack/2-Search-Space/search/stract/frontend/src/lib/optics.ts

export type OpticOption = {
  name: string;
  url: string;
  description: string;
};

export type DefaultOpticOption = OpticOption & {
  shown: boolean;
};

export const opticKey = (optic: OpticOption) => optic.name + '_' + optic.url;

export const DEFAULT_OPTICS = [
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/copycats_removal.optic',
    description: 'Remove common copycat websites from search results.',
    name: 'Copycats removal',
    shown: true,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/hacker_news.optic',
    description: 'Only return results from websites that are popular on Hacker News.',
    name: 'Hacker News',
    shown: false,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/discussions.optic',
    description: 'Only return results from forums or similar types of QA pages.',
    name: 'Discussions',
    shown: true,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/fediverse.optic',
    description: 'Only search in fediverse sites.',
    name: 'Fediverse',
    shown: false,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/indieweb_blogroll.optic',
    description:
      'Search only in the indieweb + a list of blogs from blogroll.org and some hand-picked blogs from hackernews.',
    name: 'Indieweb & blogroll',
    shown: true,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/10k_short.optic',
    description: 'Remove the top 10,000 most popular websites from search results.',
    name: '10K Short',
    shown: false,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/devdocs.optic',
    description:
      'Only return results from some of the developer documentation sites listed on devdocs.io. This is a non-exhaustive list.',
    name: 'Devdocs',
    shown: false,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/academic.optic',
    description:
      'Search exclusively in academic sites (.edu, .ac.uk, arxiv.org etc.). This is a non-exhaustive list.',
    name: 'Academic',
    shown: true,
  },
  {
    url: 'https://raw.githubusercontent.com/StractOrg/sample-optics/main/tilde.optic',
    description: 'Only search in urls that contain a tilde (~).',
    name: 'Tilde',
    shown: false,
  },
] satisfies DefaultOpticOption[];

const isPrivateIp4 = (url: string) => {
  const parts = url.split('://');
  const ip =
    parts.length > 0
      ? parts[parts.length - 1].replace(/\/$/, '').split(':')[0]
      : url.replace(/\/$/, '').split(':')[0];

  if (/^(10)\.(.*)\.(.*)\.(.*)$/.test(ip)) return true;
  if (/^(172)\.(1[6-9]|2[0-9]|3[0-1])\.(.*)\.(.*)$/.test(ip)) return true;
  if (/^(192)\.168\.(.*)\.(.*)$/.test(ip)) return true;
  if (/^(127)\.(0)\.(0)\.(1)$/.test(ip)) return true;
  if (/^(100)\.(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-7])\.(.*)\.(.*)$/.test(ip)) return true;

  return false;
};

const isPrivateIp6 = (url: string) => {
  const parts = url.split('://');
  const ip =
    parts.length > 0
      ? parts[parts.length - 1].replace(/\/$/, '').split(':')[0]
      : url.replace(/\/$/, '');

  if (/^fe80::/i.test(ip)) return true;
  if (/^fd[0-9a-f]{2}:/i.test(ip)) return true;

  return false;
};

const isPrivateIp = (url: string) => isPrivateIp4(url) || isPrivateIp6(url);

/**
 * Fetces the given `opticUrl` if allowed. The rules for which are allowed
 * should consider potentially malicious URLs such as `file://` or
 * internal/local IP addresses.
 */
export const fetchRemoteOptic = async (opts: { opticUrl: string; fetch?: typeof fetch }) => {
  if (opts.opticUrl.startsWith('file://')) return void 0;
  if (isPrivateIp(opts.opticUrl)) return void 0;

  const response = await (opts.fetch ?? fetch)(opts.opticUrl);

  if (isPrivateIp(response.url)) return void 0;

  return await response.text();
};