crabbysearch/src/cache.rs

57 lines
1.6 KiB
Rust
Raw Normal View History

//! This module provides the functionality to cache the aggregated results fetched and aggregated
//! from the upstream search engines in a json format.
use mini_moka::sync::Cache as MokaCache;
use mini_moka::sync::ConcurrentCacheExt;
✨ Compression and encryption for the cached search results (#443) * attempt1 * rough draft * add features and their optional dependancies * add encryption and compression error variants * add a sample implementation to cache trait * Update src/cache/cacher.rs Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * adjust comment so feature flag would apply? * adjust feature flag so it applies? * formatting * Update src/cache/cacher.rs update documentation Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * [features]Add base64 and chacha20 dependencies for compress-cache-results and encrypt-cache-results * move encryption key and cipher logic to separate sub module * added cacha20 and cec-results feature * added cacha20 and cec-results feature * added compression and encryption helper functions to trait implementations * added compression and encryption implementation for inMemoryCache * base64 is only requried when redis-cache feature is enabled * add error case for base64 and encryption/compression implementation to redisCache * Refactor cacher to remove regex dependency * fmt cache error and cacher * Update Cargo.toml disabling the unneeded default-features Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * fix unused import warning for mimalloc * remove deprecated method * add doc comments for encryption module * fix known bugs and use cfg-if module * make cfg-if an optional dependency * use feature-flag instead of maco lint * add comment to explain type complexity * bump app version * Update src/cache/encryption.rs Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * fixed type complexity and add docs for types --------- Co-authored-by: Spencer Najib <spencernajib2@gmail.com> Co-authored-by: alamin655 <mdalamin655@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> Co-authored-by: Spencerjibz <=spencernajib2@gmail.com> Co-authored-by: spencer <spencer@DESKTOP-SIF13AR>
2024-01-11 03:10:35 -08:00
use std::time::Duration;
2024-08-15 07:53:26 +02:00
use crate::{config::Config, models::aggregation_models::SearchResults};
2024-08-15 07:53:26 +02:00
impl From<Vec<u8>> for SearchResults {
fn from(v: Vec<u8>) -> SearchResults {
serde_json::from_slice(&v)
2024-08-14 21:34:18 +02:00
.expect("well, this can only be caused by memory corruption so good luck")
✨ Compression and encryption for the cached search results (#443) * attempt1 * rough draft * add features and their optional dependancies * add encryption and compression error variants * add a sample implementation to cache trait * Update src/cache/cacher.rs Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * adjust comment so feature flag would apply? * adjust feature flag so it applies? * formatting * Update src/cache/cacher.rs update documentation Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * [features]Add base64 and chacha20 dependencies for compress-cache-results and encrypt-cache-results * move encryption key and cipher logic to separate sub module * added cacha20 and cec-results feature * added cacha20 and cec-results feature * added compression and encryption helper functions to trait implementations * added compression and encryption implementation for inMemoryCache * base64 is only requried when redis-cache feature is enabled * add error case for base64 and encryption/compression implementation to redisCache * Refactor cacher to remove regex dependency * fmt cache error and cacher * Update Cargo.toml disabling the unneeded default-features Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * fix unused import warning for mimalloc * remove deprecated method * add doc comments for encryption module * fix known bugs and use cfg-if module * make cfg-if an optional dependency * use feature-flag instead of maco lint * add comment to explain type complexity * bump app version * Update src/cache/encryption.rs Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * fixed type complexity and add docs for types --------- Co-authored-by: Spencer Najib <spencernajib2@gmail.com> Co-authored-by: alamin655 <mdalamin655@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> Co-authored-by: Spencerjibz <=spencernajib2@gmail.com> Co-authored-by: spencer <spencer@DESKTOP-SIF13AR>
2024-01-11 03:10:35 -08:00
}
}
2024-08-15 07:53:26 +02:00
impl From<&SearchResults> for Vec<u8> {
fn from(v: &SearchResults) -> Vec<u8> {
serde_json::to_vec(v).expect("somehow failed to serialize search results")
}
}
/// Memory based cache backend.
2024-08-14 21:34:18 +02:00
#[derive(Clone)]
pub struct Cache {
/// The backend cache which stores data.
✨ Compression and encryption for the cached search results (#443) * attempt1 * rough draft * add features and their optional dependancies * add encryption and compression error variants * add a sample implementation to cache trait * Update src/cache/cacher.rs Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * adjust comment so feature flag would apply? * adjust feature flag so it applies? * formatting * Update src/cache/cacher.rs update documentation Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * [features]Add base64 and chacha20 dependencies for compress-cache-results and encrypt-cache-results * move encryption key and cipher logic to separate sub module * added cacha20 and cec-results feature * added cacha20 and cec-results feature * added compression and encryption helper functions to trait implementations * added compression and encryption implementation for inMemoryCache * base64 is only requried when redis-cache feature is enabled * add error case for base64 and encryption/compression implementation to redisCache * Refactor cacher to remove regex dependency * fmt cache error and cacher * Update Cargo.toml disabling the unneeded default-features Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * fix unused import warning for mimalloc * remove deprecated method * add doc comments for encryption module * fix known bugs and use cfg-if module * make cfg-if an optional dependency * use feature-flag instead of maco lint * add comment to explain type complexity * bump app version * Update src/cache/encryption.rs Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> * fixed type complexity and add docs for types --------- Co-authored-by: Spencer Najib <spencernajib2@gmail.com> Co-authored-by: alamin655 <mdalamin655@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: neon_arch <mustafadhuleb53@gmail.com> Co-authored-by: Spencerjibz <=spencernajib2@gmail.com> Co-authored-by: spencer <spencer@DESKTOP-SIF13AR>
2024-01-11 03:10:35 -08:00
cache: MokaCache<String, Vec<u8>>,
}
2024-08-14 21:34:18 +02:00
impl Cache {
/// Build new cache
pub fn build(config: &Config) -> Self {
log::info!("Initializing in-memory cache");
2024-08-14 21:34:18 +02:00
Self {
cache: MokaCache::builder()
2024-08-14 21:34:18 +02:00
.time_to_live(Duration::from_secs(config.cache_expiry_time))
.build(),
}
}
2024-08-14 21:34:18 +02:00
/// Retrieve Cached results
pub fn cached_results(&self, url: &str) -> Option<SearchResults> {
self.cache.get(&url.to_string()).map(|b| b.into())
}
2024-08-14 21:34:18 +02:00
/// Cache results
pub fn cache_results(&self, search_results: &[SearchResults], urls: &[String]) {
for (url, search_result) in urls.iter().zip(search_results.iter()) {
2024-08-14 21:34:18 +02:00
self.cache.insert(url.clone(), search_result.into());
}
self.cache.sync();
}
}