morrigan/crabbysearch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve code to evade bot detection and closes #8

Browse source
This commit is contained in:
neon_arch 2023-05-08 11:47:50 +03:00
parent ab7348ba3d
commit 0502a8f551
2 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/engines/duckduckgo.rs
View file

@ -4,7 +4,7 @@
use std::collections::HashMap;
use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT};
use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
use scraper::{Html, Selector};
use crate::search_results_handler::aggregation_models::RawSearchResult;
@ -50,7 +50,8 @@ pub async fn results(
let mut header_map = HeaderMap::new();
header_map.insert(USER_AGENT, user_agent.parse()?);
header_map.insert(REFERER, "https://google.com/".parse()?);
header_map.insert(CONTENT_TYPE, "text/html; charset=UTF-8".parse()?);
header_map.insert(CONTENT_TYPE, "application/x-www-form-urlencoded".parse()?);
header_map.insert(COOKIE, "kl=wt-wt".parse()?);
// fetch the html from upstream duckduckgo engine
// TODO: Write better error handling code to handle no results case.