app/api/langchain-tools/duckduckgo_search.ts

import { decode } from "html-entities";
import { convert as htmlToText } from "html-to-text";
import { Tool } from "langchain/tools";

const SEARCH_REGEX =
  /DDG\.pageLayout\.load\('d',(\[.+\])\);DDG\.duckbar\.load\('images'/;
const IMAGES_REGEX =
  /;DDG\.duckbar\.load\('images', ({"ads":.+"vqd":{".+":"\d-\d+-\d+"}})\);DDG\.duckbar\.load\('news/;
const NEWS_REGEX =
  /;DDG\.duckbar\.load\('news', ({"ads":.+"vqd":{".+":"\d-\d+-\d+"}})\);DDG\.duckbar\.load\('videos/;
const VIDEOS_REGEX =
  /;DDG\.duckbar\.load\('videos', ({"ads":.+"vqd":{".+":"\d-\d+-\d+"}})\);DDG\.duckbar\.loadModule\('related_searches/;
const RELATED_SEARCHES_REGEX =
  /DDG\.duckbar\.loadModule\('related_searches', ({"ads":.+"vqd":{".+":"\d-\d+-\d+"}})\);DDG\.duckbar\.load\('products/;
const VQD_REGEX = /vqd=['"](\d+-\d+(?:-\d+)?)['"]/;

interface CallbackSearchResult {
  /** Website description */
  a: string;
  /** Unknown */
  ae: null;
  /** ddg!bang information (ex. w Wikipedia en.wikipedia.org) */
  b?: string;
  /** URL */
  c: string;
  /** URL of some sort. */
  d: string;
  /** Class name associations. */
  da?: string;
  /** Unknown */
  h: number;
  /** Website hostname */
  i: string;
  /** Unknown */
  k: null;
  /** Unknown */
  m: number;
  /** Unknown */
  o: number;
  /** Unknown */
  p: number;
  /** Unknown */
  s: string;
  /** Website Title */
  t: string;
  /** Website URL */
  u: string;
}

interface CallbackNextSearch {
  /** URL to the next page of results */
  n: string;
}

interface CallbackDuckbarPayload<T> {
  ads: null | any[];
  query: string;
  queryEncoded: string;
  response_type: string;
  results: T[];
  vqd: {
    [query: string]: string;
  };
}

interface DuckbarImageResult {
  /** The height of the image in pixels. */
  height: number;
  /** The image URL. */
  image: string;
  /** The source of the image. */
  source: string;
  /** The thumbnail URL. */
  thumbnail: string;
  /** The title (or caption) of the image. */
  title: string;
  /** The website URL of where the image came from. */
  url: string;
  /** The width of the image in pixels. */
  width: number;
}

interface DuckbarVideoResult {
  /** URL of the video */
  content: string;
  /** Description of the video */
  description: string;
  /** Duration of the video */
  duration: string;
  /** Embed HTML for the video */
  embed_html: string;
  /** Embed URL for the video */
  embed_url: string;
  /** Thumbnail images of the video */
  images: {
    large: string;
    medium: string;
    motion: string;
    small: string;
  };
  /** Where this search result came from */
  provider: string;
  /** ISO timestamp of the upload */
  published: string;
  /** What site the video was on */
  publisher: string;
  /** Various statistics */
  statistics: {
    /** View count of the video */
    viewCount: number | null;
  };
  /** Title of the video */
  title: string;
  /** Name of the video uploader(?) */
  uploader: string;
}

interface DuckbarRelatedSearch {
  display_text: string;
  text: string;
  web_search_url: string;
}

interface DuckbarNewsResult {
  date: number;
  excerpt: string;
  image?: string;
  relative_time: string;
  syndicate: string;
  title: string;
  url: string;
  use_relevancy: number;
  is_old?: number;
  fetch_image?: number;
}

interface SearchResults {
  /** Whether there were no results found. */
  noResults: boolean;
  /** The VQD of the search query. */
  vqd: string;
  /** The web results of the search. */
  results: SearchResult[];
  /** The image results of the search. */
  images?: DuckbarImageResult[];
  /** The news article results of the search. */
  news?: NewsResult[];
  /** The video results of the search. */
  videos?: VideoResult[];
  /** The related searches of the query. */
  related?: RelatedResult[];
}

interface VideoResult {
  /** The URL of the video. */
  url: string;
  /** The title of the video. */
  title: string;
  /** The description of the video. */
  description: string;
  /** The image URL of the video. */
  image: string;
  /** The duration of the video. (i.e. "9:20") */
  duration: string;
  /** The ISO timestamp of when the video was published. */
  published: string;
  /** Where the video was publised on. (i.e. "YouTube") */
  publishedOn: string;
  /** The name of who uploaded the video. */
  publisher: string;
  /** The view count of the video. */
  viewCount?: number;
}

interface NewsResult {
  /** The timestamp of when the article was created. */
  date: number;
  /** An except of the article. */
  excerpt: string;
  /** The image URL used in the article. */
  image?: string;
  /** The relative time of when the article was posted, in human readable format. */
  relativeTime: string;
  /** Where this article was indexed from. */
  syndicate: string;
  /** The title of the article. */
  title: string;
  /** The URL of the article. */
  url: string;
  /** Whether this article is classified as old. */
  isOld: boolean;
}

interface SearchResult {
  /** The hostname of the website. (i.e. "google.com") */
  hostname: string;
  /** The URL of the result. */
  url: string;
  /** The title of the result. */
  title: string;
  /**
   * The sanitized description of the result.
   * Bold tags will still be present in this string.
   */
  description: string;
  /** The description of the result. */
  rawDescription: string;
  /** The icon of the website. */
  icon: string;
  /** The ddg!bang information of the website, if any. */
  bang?: SearchResultBang;
}

interface SearchResultBang {
  /** The prefix of the bang. (i.e. "w" for !w) */
  prefix: string;
  /** The title of the bang. */
  title: string;
  /** The domain of the bang. */
  domain: string;
}

interface RelatedResult {
  text: string;
  raw: string;
}

enum SearchTimeType {
  /** From any time. */
  ALL = "a",
  /** From the past day. */
  DAY = "d",
  /** From the past week. */
  WEEK = "w",
  /** From the past month. */
  MONTH = "m",
  /** From the past year. */
  YEAR = "y",
}

interface SearchOptions {
  /** The safe search type of the search. */
  safeSearch?: SafeSearchType;
  /** The time range of the searches, can be a SearchTimeType or a date range ("2021-03-16..2021-03-30") */
  time?: SearchTimeType | string;
  /** The locale(?) of the search. Defaults to "en-us". */
  locale?: string;
  /** The region of the search. Defaults to "wt-wt" or all regions. */
  region?: string;
  /** The market region(?) of the search. Defaults to "US". */
  marketRegion?: string;
  /** The number to offset the results to. */
  offset?: number;
  /**
   * The string that acts like a key to a search.
   * Set this if you made a search with the same query.
   */
  vqd?: string;
}

enum SafeSearchType {
  /** Strict filtering, no NSFW content. */
  STRICT = 0,
  /** Moderate filtering. */
  MODERATE = -1,
  /** No filtering. */
  OFF = -2,
}

const defaultOptions: SearchOptions = {
  safeSearch: SafeSearchType.OFF,
  time: SearchTimeType.ALL,
  locale: "en-us",
  region: "wt-wt",
  offset: 0,
  marketRegion: "us",
};

async function search(
  query: string,
  options?: SearchOptions,
): Promise<SearchResults> {
  if (!query) throw new Error("Query cannot be empty!");
  if (!options) options = defaultOptions;
  else options = sanityCheck(options);

  let vqd = options.vqd!;
  if (!vqd) vqd = await getVQD(query, "web");

  const queryObject: Record<string, string> = {
    q: query,
    ...(options.safeSearch !== SafeSearchType.STRICT ? { t: "D" } : {}),
    l: options.locale!,
    ...(options.safeSearch === SafeSearchType.STRICT ? { p: "1" } : {}),
    kl: options.region || "wt-wt",
    s: String(options.offset),
    dl: "en",
    ct: "US",
    ss_mkt: options.marketRegion!,
    df: options.time! as string,
    vqd,
    ...(options.safeSearch !== SafeSearchType.STRICT
      ? { ex: String(options.safeSearch) }
      : {}),
    sp: "1",
    bpa: "1",
    biaexp: "b",
    msvrtexp: "b",
    ...(options.safeSearch === SafeSearchType.STRICT
      ? {
          videxp: "a",
          nadse: "b",
          eclsexp: "a",
          stiaexp: "a",
          tjsexp: "b",
          related: "b",
          msnexp: "a",
        }
      : {
          nadse: "b",
          eclsexp: "b",
          tjsexp: "b",
          // cdrexp: 'b'
        }),
  };

  const response = await fetch(
    `https://links.duckduckgo.com/d.js?${queryString(queryObject)}`,
  );
  const data = await response.text();

  if (data.includes("DDG.deep.is506"))
    throw new Error("A server error occurred!");

  const searchResults = JSON.parse(
    SEARCH_REGEX.exec(data)![1].replace(/\t/g, "    "),
  ) as (CallbackSearchResult | CallbackNextSearch)[];

  if (searchResults.length === 1 && !("n" in searchResults[0])) {
    const onlyResult = searchResults[0] as CallbackSearchResult;
    /* istanbul ignore next */
    if (
      (!onlyResult.da && onlyResult.t === "EOF") ||
      !onlyResult.a ||
      onlyResult.d === "google.com search"
    )
      return {
        noResults: true,
        vqd,
        results: [],
      };
  }

  const results: SearchResults = {
    noResults: false,
    vqd,
    results: [],
  };

  for (const search of searchResults) {
    if ("n" in search) continue;
    let bang: SearchResultBang | undefined;
    if (search.b) {
      const [prefix, title, domain] = search.b.split("\t");
      bang = { prefix, title, domain };
    }
    results.results.push({
      title: search.t,
      description: decode(search.a),
      rawDescription: search.a,
      hostname: search.i,
      icon: `https://external-content.duckduckgo.com/ip3/${search.i}.ico`,
      url: search.u,
      bang,
    });
  }

  // Images
  const imagesMatch = IMAGES_REGEX.exec(data);
  if (imagesMatch) {
    const imagesResult = JSON.parse(
      imagesMatch[1].replace(/\t/g, "    "),
    ) as CallbackDuckbarPayload<DuckbarImageResult>;
    results.images = imagesResult.results.map((i) => {
      i.title = decode(i.title);
      return i;
    });
  }

  // News
  const newsMatch = NEWS_REGEX.exec(data);
  if (newsMatch) {
    const newsResult = JSON.parse(
      newsMatch[1].replace(/\t/g, "    "),
    ) as CallbackDuckbarPayload<DuckbarNewsResult>;
    results.news = newsResult.results.map((article) => ({
      date: article.date,
      excerpt: decode(article.excerpt),
      image: article.image,
      relativeTime: article.relative_time,
      syndicate: article.syndicate,
      title: decode(article.title),
      url: article.url,
      isOld: !!article.is_old,
    })) as NewsResult[];
  }

  // Videos
  const videosMatch = VIDEOS_REGEX.exec(data);
  if (videosMatch) {
    const videoResult = JSON.parse(
      videosMatch[1].replace(/\t/g, "    "),
    ) as CallbackDuckbarPayload<DuckbarVideoResult>;
    results.videos = [];
    /* istanbul ignore next */
    for (const video of videoResult.results) {
      results.videos.push({
        url: video.content,
        title: decode(video.title),
        description: decode(video.description),
        image:
          video.images.large ||
          video.images.medium ||
          video.images.small ||
          video.images.motion,
        duration: video.duration,
        publishedOn: video.publisher,
        published: video.published,
        publisher: video.uploader,
        viewCount: video.statistics.viewCount || undefined,
      });
    }
  }

  // Related Searches
  const relatedMatch = RELATED_SEARCHES_REGEX.exec(data);
  if (relatedMatch) {
    const relatedResult = JSON.parse(
      relatedMatch[1].replace(/\t/g, "    "),
    ) as CallbackDuckbarPayload<DuckbarRelatedSearch>;
    results.related = [];
    for (const related of relatedResult.results) {
      results.related.push({
        text: related.text,
        raw: related.display_text,
      });
    }
  }
  return results;
}

function queryString(query: Record<string, string>) {
  return new URLSearchParams(query).toString();
}

async function getVQD(query: string, ia = "web") {
  try {
    const response = await fetch(
      `https://duckduckgo.com/?${queryString({ q: query, ia })}`,
    );
    const data = await response.text();
    return VQD_REGEX.exec(data)![1];
  } catch (e) {
    throw new Error(`Failed to get the VQD for query "${query}".`);
  }
}

function sanityCheck(options: SearchOptions) {
  options = Object.assign({}, defaultOptions, options);

  if (!(options.safeSearch! in SafeSearchType))
    throw new TypeError(
      `${options.safeSearch} is an invalid safe search type!`,
    );

  /* istanbul ignore next */
  if (typeof options.safeSearch! === "string")
    options.safeSearch = SafeSearchType[
      options.safeSearch!
    ] as any as SafeSearchType;

  if (typeof options.offset !== "number")
    throw new TypeError(`Search offset is not a number!`);

  if (options.offset! < 0)
    throw new RangeError("Search offset cannot be below zero!");

  if (
    options.time &&
    !Object.values(SearchTimeType).includes(options.time as SearchTimeType) &&
    !/\d{4}-\d{2}-\d{2}..\d{4}-\d{2}-\d{2}/.test(options.time as string)
  )
    throw new TypeError(`${options.time} is an invalid search time!`);

  if (!options.locale || typeof options.locale! !== "string")
    throw new TypeError("Search locale must be a string!");

  if (!options.region || typeof options.region! !== "string")
    throw new TypeError("Search region must be a string!");

  if (!options.marketRegion || typeof options.marketRegion! !== "string")
    throw new TypeError("Search market region must be a string!");

  if (options.vqd && !/\d-\d+-\d+/.test(options.vqd))
    throw new Error(`${options.vqd} is an invalid VQD!`);

  return options;
}

export class DuckDuckGo extends Tool {
  name = "duckduckgo_search";
  maxResults = 4;

  /** @ignore */
  async _call(input: string) {
    const searchResults = await search(input, {
      safeSearch: SafeSearchType.OFF,
    });

    if (searchResults.noResults) {
      return "No good search result found";
    }

    const results = searchResults.results
      .slice(0, this.maxResults)
      .map(({ title, description, url }) => htmlToText(description))
      .join("\n\n");

    return results;
  }

  description =
    "a search engine. useful for when you need to answer questions about current events. input should be a search query.";
}