morrigan/crabbysearch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'rolling' into FIX/463_results_from_different_search_engines_get_cached_as_the_same_key

Browse source
This commit is contained in:
neon_arch 2024-01-02 12:29:54 +03:00 • committed by GitHub
commit 36e2ac93be
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 206 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

124
src/engines/bing.rs Normal file
View file

@ -0,0 +1,124 @@
//! The `bing` module handles the scraping of results from the bing search engine
//! by querying the upstream bing search engine with user provided query and with a page
//! number if provided.
use std::collections::HashMap;
use regex::Regex;
use reqwest::header::HeaderMap;
use reqwest::Client;
use scraper::Html;
use crate::models::aggregation_models::SearchResult;
use crate::models::engine_models::{EngineError, SearchEngine};
use error_stack::{Report, Result, ResultExt};
use super::search_result_parser::SearchResultParser;
/// A new Bing engine type defined in-order to implement the `SearchEngine` trait which allows to
/// reduce code duplication as well as allows to create vector of different search engines easily.
pub struct Bing {
/// The parser, used to interpret the search result.
parser: SearchResultParser,
}
impl Bing {
/// Creates the Bing parser.
pub fn new() -> Result<Self, EngineError> {
Ok(Self {
parser: SearchResultParser::new(
".b_results",
".b_algo",
"h2 a",
".tpcn a.tilk",
".b_caption p",
)?,
})
}
}
#[async_trait::async_trait]
impl SearchEngine for Bing {
async fn results(
&self,
query: &str,
page: u32,
user_agent: &str,
client: &Client,
_safe_search: u8,
) -> Result<HashMap<String, SearchResult>, EngineError> {
// Bing uses `start results from this number` convention
// So, for 10 results per page, page 0 starts at 1, page 1
// starts at 11, and so on.
let results_per_page = 10;
let start_result = results_per_page * page + 1;
let url: String = match page {
0 => {
format!("https://www.bing.com/search?q={query}")
}
_ => {
format!("https://www.bing.com/search?q={query}&first={start_result}")
}
};
let query_params: Vec<(&str, &str)> = vec![
("_EDGE_V", "1"),
("SRCHD=AF", "NOFORM"),
("_Rwho=u", "d"),
("bngps=s", "0"),
("_UR=QS=0&TQS", "0"),
("_UR=QS=0&TQS", "0"),
];
let mut cookie_string = String::new();
for (k, v) in &query_params {
cookie_string.push_str(&format!("{k}={v}; "));
}
let header_map = HeaderMap::try_from(&HashMap::from([
("USER_AGENT".to_string(), user_agent.to_string()),
("REFERER".to_string(), "https://google.com/".to_string()),
(
"CONTENT_TYPE".to_string(),
"application/x-www-form-urlencoded".to_string(),
),
("COOKIE".to_string(), cookie_string),
]))
.change_context(EngineError::UnexpectedError)?;
let document: Html = Html::parse_document(
&Bing::fetch_html_from_upstream(self, &url, header_map, client).await?,
);
// Bing is very aggressive in finding matches
// even with the most absurd of queries. ".b_algo" is the
// class for the list item of results
if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(0) {
if no_result_msg
.value()
.attr("class")
.map(|classes| classes.contains("b_algo"))
.unwrap_or(false)
{
return Err(Report::new(EngineError::EmptyResultSet));
}
}
let re_span = Regex::new(r#"<span.*?>.*?(?:</span>&nbsp;·|</span>)"#).unwrap();
let re_strong = Regex::new(r#"(<strong>|</strong>)"#).unwrap();
// scrape all the results from the html
self.parser
.parse_for_results(&document, |title, url, desc| {
Some(SearchResult::new(
&re_strong.replace_all(title.inner_html().trim(), ""),
url.value().attr("href").unwrap(),
&re_span.replace_all(desc.inner_html().trim(), ""),
&["bing"],
))
})
}
}

1
src/engines/mod.rs
View file

@ -3,6 +3,7 @@
//! provide a standard functions to be implemented for all the upstream search engine handling
//! code. Moreover, it also provides a custom error for the upstream search engine handling code.
pub mod bing;
pub mod brave;
pub mod duckduckgo;
pub mod librex;

9
src/lib.rs
View file

@ -21,7 +21,12 @@ use crate::server::router;
use actix_cors::Cors;
use actix_files as fs;
use actix_governor::{Governor, GovernorConfigBuilder};
use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer};
use actix_web::{
dev::Server,
http::header,
middleware::{Compress, Logger},
web, App, HttpServer,
};
use cache::cacher::{Cacher, SharedCache};
use config::parser::Config;
use handler::{file_path, FileType};
@ -73,6 +78,8 @@ pub fn run(
]);
App::new()
// Compress the responses provided by the server for the client requests.
.wrap(Compress::default())
.wrap(Logger::default()) // added logging middleware for logging.
.app_data(web::Data::new(config.clone()))
.app_data(cache.clone())

4
src/models/engine_models.rs
View file

@ -166,6 +166,10 @@ impl EngineHandler {
let engine = crate::engines::mojeek::Mojeek::new()?;
("mojeek", Box::new(engine))
}
"bing" => {
let engine = crate::engines::bing::Bing::new()?;
("bing", Box::new(engine))
}
_ => {
return Err(Report::from(EngineError::NoSuchEngineFound(
engine_name.to_string(),

80
src/server/router.rs
View file

@ -6,22 +6,20 @@ use crate::{
config::parser::Config,
handler::{file_path, FileType},
};
use actix_web::{get, web, HttpRequest, HttpResponse};
use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse};
use std::fs::read_to_string;
/// Handles the route of index page or main page of the `websurfx` meta search engine website.
#[get("/")]
pub async fn index(config: web::Data<Config>) -> Result<HttpResponse, Box<dyn std::error::Error>> {
Ok(HttpResponse::Ok()
.content_type("text/html; charset=utf-8")
.body(
crate::templates::views::index::index(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
)
.0,
))
Ok(HttpResponse::Ok().content_type(ContentType::html()).body(
crate::templates::views::index::index(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
)
.0,
))
}
/// Handles the route of any other accessed route/page which is not provided by the
@ -29,16 +27,14 @@ pub async fn index(config: web::Data<Config>) -> Result<HttpResponse, Box<dyn st
pub async fn not_found(
config: web::Data<Config>,
) -> Result<HttpResponse, Box<dyn std::error::Error>> {
Ok(HttpResponse::Ok()
.content_type("text/html; charset=utf-8")
.body(
crate::templates::views::not_found::not_found(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
)
.0,
))
Ok(HttpResponse::Ok().content_type(ContentType::html()).body(
crate::templates::views::not_found::not_found(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
)
.0,
))
}
/// Handles the route of robots.txt page of the `websurfx` meta search engine website.
@ -47,23 +43,21 @@ pub async fn robots_data(_req: HttpRequest) -> Result<HttpResponse, Box<dyn std:
let page_content: String =
read_to_string(format!("{}/robots.txt", file_path(FileType::Theme)?))?;
Ok(HttpResponse::Ok()
.content_type("text/plain; charset=ascii")
.content_type(ContentType::plaintext())
.body(page_content))
}
/// Handles the route of about page of the `websurfx` meta search engine website.
#[get("/about")]
pub async fn about(config: web::Data<Config>) -> Result<HttpResponse, Box<dyn std::error::Error>> {
Ok(HttpResponse::Ok()
.content_type("text/html; charset=utf-8")
.body(
crate::templates::views::about::about(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
)
.0,
))
Ok(HttpResponse::Ok().content_type(ContentType::html()).body(
crate::templates::views::about::about(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
)
.0,
))
}
/// Handles the route of settings page of the `websurfx` meta search engine website.
@ -71,16 +65,14 @@ pub async fn about(config: web::Data<Config>) -> Result<HttpResponse, Box<dyn st
pub async fn settings(
config: web::Data<Config>,
) -> Result<HttpResponse, Box<dyn std::error::Error>> {
Ok(HttpResponse::Ok()
.content_type("text/html; charset=utf-8")
.body(
crate::templates::views::settings::settings(
config.safe_search,
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
&config.upstream_search_engines,
)?
.0,
))
Ok(HttpResponse::Ok().content_type(ContentType::html()).body(
crate::templates::views::settings::settings(
config.safe_search,
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
&config.upstream_search_engines,
)?
.0,
))
}

24
src/server/routes/search.rs
View file

@ -11,7 +11,7 @@ use crate::{
},
results::aggregator::aggregate,
};
use actix_web::{get, web, HttpRequest, HttpResponse};
use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse};
use regex::Regex;
use std::{
fs::File,
@ -68,18 +68,16 @@ pub async fn search(
get_results(page + 1)
);
Ok(HttpResponse::Ok()
.content_type("text/html; charset=utf-8")
.body(
crate::templates::views::search::search(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
query,
&results?,
)
.0,
))
Ok(HttpResponse::Ok().content_type(ContentType::html()).body(
crate::templates::views::search::search(
&config.style.colorscheme,
&config.style.theme,
&config.style.animation,
query,
&results?,
)
.0,
))
}
None => Ok(HttpResponse::TemporaryRedirect()
.insert_header(("location", "/"))