Scraping Linkedin Search Results

index.js

// copy into a file called linkedinScrape.js (or whatever you want to call it) then call node linkedinScrape.js 
const cookies = "" // your cookies
const page = 1
const searchTerm = "marketing agency"
const results = await searchLinkedIn(searchTerm, page)
console.log(results)


export async function searchLinkedIn(query, page = 1) {
  try {
    const search = await searchPeople(query, page);
    const profiles = jsonifyLinkedinSearchResults(search);
    return profiles;
  } catch (error) {
    console.log("error at search: ", error.message);
  }
}

async function searchPeople(search, page = 1) {
  try {
    const start = (page - 1) * 10;
    const res = await fetch(
      `https://www.linkedin.com/voyager/api/graphql?variables=(start:${start},origin:SWITCH_SEARCH_VERTICAL,query:(keywords:${encodeURIComponent(
        search
      )},flagshipSearchIntent:SEARCH_SRP,queryParameters:List((key:heroEntityKey,value:List(urn%3Ali%3Aautocomplete%3A1052861661)),(key:position,value:List(1)),(key:resultType,value:List(PEOPLE)),(key:searchId,value:List(06c4ced8-68b0-4f81-902e-1e71147a775b))),includeFiltersInResponse:false))&&queryId=voyagerSearchDashClusters.181547298141ca2c72182b748713641b`,
      {
        headers: {
          accept: "application/vnd.linkedin.normalized+json+2.1",
          "accept-language": "en-US,en-CA;q=0.9,en-AU;q=0.8,en;q=0.7",
          "csrf-token": "ajax:1690738384797705558",
          "sec-ch-ua":
            '"Chromium";v="112", "Google Chrome";v="112", "Not:A-Brand";v="99"',
          "sec-ch-ua-mobile": "?0",
          "sec-ch-ua-platform": '"macOS"',
          "sec-fetch-dest": "empty",
          "sec-fetch-mode": "cors",
          "sec-fetch-site": "same-origin",
          "x-li-lang": "en_US",
          "x-li-page-instance":
            "urn:li:page:d_flagship3_search_srp_people;WCNsJsQPSo63RBfjtgGw3Q==",
          "x-li-track":
            '{"clientVersion":"1.12.3124","mpVersion":"1.12.3124","osName":"web","timezoneOffset":-5,"timezone":"America/Chicago","deviceFormFactor":"DESKTOP","mpName":"voyager-web","displayDensity":1,"displayWidth":1920,"displayHeight":1080}',
          "x-restli-protocol-version": "2.0.0",
          cookie: cookies,
          Referer:
            "https://www.linkedin.com/search/results/all/?fetchDeterministicClustersOnly=false&heroEntityKey=urn%3Ali%3Aautocomplete%3A1052861661&keywords=software%20engineer&origin=AUTO_COMPLETE&position=1&searchId=06c4ced8-68b0-4f81-902e-1e71147a775b&sid=%3Bd%40",
          "Referrer-Policy": "strict-origin-when-cross-origin",
        },
        body: null,
        method: "GET",
      }
    );
    const json = await res.json();
    if (json?.status === 429) {
      console.log("429 errors!!! Shut it down!");
      return json;
    }
    const error = json?.data?.errors;
    if (error) {
      console.log("error at search: ", error);
    }
    return json;
  } catch (error) {
    console.log("error at search: ", error.message);
  }
}

function getHandle(linkedinProfileUrl) {
  const url = getLinkedinProfileUrl(linkedinProfileUrl);
  return url?.split("/in/")?.[1];
}

function getLinkedinProfileUrl(navigationUrl) {
  return navigationUrl?.split("?")?.[0];
}

function getCompanyName(profile) {
  const summaryCompanyName = profile?.summary?.text?.split(" at ")[1];
  if (!summaryCompanyName) {
    const profileCompanyName = profile?.primarySubtitle?.text?.split(" at ")[1];
    return profileCompanyName;
  }
  return summaryCompanyName;
}

export function jsonifyLinkedinSearchResults(json) {
  const peoplesProfiles = json?.included?.filter(
    (s) => s?.template === "UNIVERSAL"
  );
  const jsonify = peoplesProfiles
    .map((p) => {
      return {
        id: p?.trackingUrn?.split(":")?.[3],
        name: p?.title?.text,
        handle: getHandle(p?.navigationUrl),
        jobTitle: p?.primarySubtitle?.text,
        summary: p?.summary?.text,
        company: getCompanyName(p),
        url: getLinkedinProfileUrl(p?.navigationUrl),
        location: p?.secondarySubtitle?.text,
        image:
          p?.image?.attributes?.[0]?.detailData?.nonEntityProfilePicture
            ?.vectorImage?.artifacts?.[0]?.fileIdentifyingUrlPathSegment,
      };
    })
    .filter((p) => p?.id !== "headless");
  return jsonify;
}