Modified parse function

- Rewrite parse fucntion.
- Implement Error handling using derive_error crate.
- Create new RaPostItems to store items related to post.
This commit is contained in:
Uttarayan Mondal
2021-03-16 03:19:56 +05:30
parent c95dc097d4
commit 52d70d99c4
3 changed files with 88 additions and 81 deletions
+2
View File
@@ -8,7 +8,9 @@ edition = "2018"
[dependencies] [dependencies]
derive-error = "0.0.5"
reqwest = "0.11.*" reqwest = "0.11.*"
chrono = "0.4.*" chrono = "0.4.*"
json = "0.12.*" json = "0.12.*"
tokio = { version = "1.3.*", features = ["full"] } tokio = { version = "1.3.*", features = ["full"] }
+10 -7
View File
@@ -7,30 +7,33 @@
//!extern crate tokio; //!extern crate tokio;
//!use crate::rapr::{RaprClient,RaSub}; //!use crate::rapr::{RaprClient,RaSub};
//!#[tokio::main] //!#[tokio::main]
//!async fn main() { //!async fn main() -> Result<(), rapr::Error> {
//! let client = RaprClient::new(); //! let client = RaprClient::new();
//! let mut sub = RaSub::new("pics"); //! let mut sub = RaSub::new("pics");
//! client.fetch(10, &mut sub).await.unwrap(); //! client.fetch(10, &mut sub).await?;
//! for post in sub.posts { //! for post in sub.posts {
//! println!("{}",post.title); //! println!("{}",post.title);
//! } //! }
//! Ok(())
//!} //!}
#[macro_use]
extern crate derive_error;
mod rapr; mod rapr;
pub use crate::rapr::{RaPost, RaSub, RaprClient}; pub use crate::rapr::Error;
pub use crate::rapr::{RaPost, RaPostItems, RaSub, RaprClient};
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::rapr::{RaSub, RaprClient}; use crate::rapr::{RaSub, RaprClient};
#[tokio::test] #[tokio::test]
async fn subreddit() { async fn subreddit() -> Result<(), crate::rapr::Error> {
let client = RaprClient::new(); let client = RaprClient::new();
let mut sub = RaSub::new("rust"); let mut sub = RaSub::new("rust");
client.fetch(10, &mut sub).await.unwrap(); client.fetch(10, &mut sub).await?;
// println!("{:#?}", sub.posts.len());
// client.fetch(5, &mut sub).await.unwrap();
for post in sub.posts { for post in sub.posts {
println!("{:#?}", post); println!("{:#?}", post);
} }
Ok(())
} }
#[tokio::test] #[tokio::test]
async fn title() { async fn title() {
+76 -74
View File
@@ -1,20 +1,38 @@
extern crate json; extern crate json;
extern crate reqwest; extern crate reqwest;
extern crate tokio; extern crate tokio;
use chrono::{DateTime, Local}; // use chrono::{DateTime, Local};
use std::fmt; use std::fmt;
#[derive(Debug)] /// Error enum
#[derive(Debug, Error)]
pub enum Error { pub enum Error {
JsonParseError, UnexpectedJson,
HttpGetError, NoneError,
JsonParseError(json::Error),
RequestError(reqwest::Error),
} }
#[derive(Debug, Clone, Copy)] /// Items related to the post
pub enum TextType { #[derive(Debug, Clone)]
HTML, pub struct RaPostItems {
Raw, pub upvotes: u32,
Empty, pub downvotes: u32,
pub permalink: String,
pub url: Option<String>,
}
impl RaPostItems {
/// Create a new struct of items for a post
/// You probably don't need this fucntion.
pub fn new(upvotes: u32, downvotes: u32, permalink: &str, url: Option<String>) -> Self {
Self {
upvotes,
downvotes,
permalink: permalink.to_string(),
url,
}
}
} }
/// Reddit post object /// Reddit post object
@@ -23,13 +41,9 @@ pub enum TextType {
#[derive(Clone)] #[derive(Clone)]
pub struct RaPost { pub struct RaPost {
pub id: String, pub id: String,
pub datetime: DateTime<Local>,
pub title: String, pub title: String,
pub text: Option<String>, pub text: Option<String>,
texttype: TextType, pub items: RaPostItems,
pub permalink: String,
pub upvotes: u32,
pub downvotes: u32,
pub json: json::JsonValue, pub json: json::JsonValue,
} }
@@ -37,12 +51,10 @@ impl fmt::Debug for RaPost {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RaPost") f.debug_struct("RaPost")
.field("id", &self.id) .field("id", &self.id)
.field("datetime", &self.datetime)
.field("title", &self.title) .field("title", &self.title)
.field("text", &self.text) .field("text", &self.text)
.field("texttype", &self.texttype) .field("permalink", &self.items.permalink)
.field("upvotes", &self.upvotes) .field("url", &self.items.url)
.field("downvotes", &self.downvotes)
.finish() .finish()
} }
} }
@@ -53,54 +65,35 @@ impl RaPost {
id: &str, id: &str,
title: &str, title: &str,
text: Option<&str>, text: Option<&str>,
texttype: TextType, items: RaPostItems,
permalink: &str,
upvotes: u32,
downvotes: u32,
json: &json::JsonValue, json: &json::JsonValue,
) -> Self { ) -> Self {
let _text = match text {
Some(text) => Some(text.to_string()),
None => None,
};
Self { Self {
id: id.to_string(), id: id.to_string(),
title: title.to_string(), title: title.to_string(),
datetime: Local::now(), // Temp text: text.map(String::from),
text: _text, items,
texttype,
permalink: String::from(permalink),
upvotes,
downvotes,
json: json.clone(), json: json.clone(),
} }
} }
/// Parse json from `json['data']['children']` array elements. /// Parse json from `json['data']['children']` array elements.
pub fn parse(post: &json::JsonValue) -> Result<RaPost, json::Error> { pub fn parse(post: &json::JsonValue) -> Result<RaPost, Error> {
let mut text: Option<&str> = post["selftext"].as_str(); let id = post["name"].as_str().ok_or(Error::UnexpectedJson)?;
let mut texttype: TextType = TextType::Empty; let title = post["title"].as_str().ok_or(Error::UnexpectedJson)?;
// Reddit always returns an empty string on selftext let upvotes = post["ups"].as_u32().ok_or(Error::UnexpectedJson)?;
// if there is no text. So this shouldn't panic! let downvotes = post["downs"].as_u32().ok_or(Error::UnexpectedJson)?;
// Some also have both selftext and selftext_html let permalink = post["permalink"].as_str().ok_or(Error::UnexpectedJson)?;
// I am only taking selftext from these. let url = post["url"].as_str().map(String::from);
if text.unwrap().is_empty() {
text = post["selftext_html"].as_str(); let mut selftext = post["selftext"].as_str();
if text != None { if selftext.ok_or(Error::NoneError)?.is_empty() {
texttype = TextType::HTML; // Either unwraps and sets new value or reamins ""
} selftext = post["selftext_html"].as_str();
} else {
texttype = TextType::Raw;
} }
Ok(RaPost::new( let items = RaPostItems::new(upvotes, downvotes, permalink, url);
post["name"].as_str().unwrap(),
post["title"].as_str().unwrap(), Ok(RaPost::new(id, title, selftext, items, post))
text,
texttype,
post["permalink"].as_str().unwrap(),
post["ups"].as_u32().unwrap(),
post["downs"].as_u32().unwrap(),
post,
))
} }
} }
@@ -134,6 +127,12 @@ pub struct RaprClient {
rwclient: reqwest::Client, rwclient: reqwest::Client,
} }
impl Default for RaprClient {
fn default() -> Self {
Self::new()
}
}
impl RaprClient { impl RaprClient {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
@@ -141,6 +140,9 @@ impl RaprClient {
rwclient: reqwest::Client::new(), rwclient: reqwest::Client::new(),
} }
} }
pub fn default() -> Self {
Self::new()
}
/// Fetch posts from subreddit and store them in the subreddit object. /// Fetch posts from subreddit and store them in the subreddit object.
/// Note: First fetch always seems to pull two pinned posts which are not marked pinned in the json /// Note: First fetch always seems to pull two pinned posts which are not marked pinned in the json
@@ -151,34 +153,34 @@ impl RaprClient {
}; };
let res = match &sub.after { let res = match &sub.after {
None => self None => {
.rwclient self.rwclient
.get(url) .get(url)
.query(&[("limit", count)]) .query(&[("limit", count)])
.send() .send()
.await .await?
.unwrap(), }
Some(after) => self Some(after) => {
.rwclient self.rwclient
.get(url) .get(url)
.query(&[("limit", count.to_string()), ("after", after.to_string())]) .query(&[("limit", count.to_string()), ("after", after.to_string())])
.send() .send()
.await .await?
.unwrap(), }
}; };
let mut parsed = json::parse(res.text().await.unwrap().as_str()).unwrap(); let mut parsed = json::parse(res.text().await?.as_str())?;
let raw_posts: Vec<json::JsonValue> = match parsed["data"]["children"].take() { let raw_posts: Vec<json::JsonValue> = match parsed["data"]["children"].take() {
json::JsonValue::Array(arr) => arr, json::JsonValue::Array(arr) => arr,
_ => return Err(Error::JsonParseError), _ => return Err(Error::UnexpectedJson),
}; };
let mut parsed_posts: Vec<RaPost> = Vec::new(); let mut parsed_posts: Vec<RaPost> = Vec::new();
for post in raw_posts { for post in raw_posts {
if post["kind"].as_str().unwrap() == "t3" { if post["kind"].as_str().ok_or(Error::NoneError)? == "t3" {
parsed_posts.push(RaPost::parse(&post["data"]).unwrap()); parsed_posts.push(RaPost::parse(&post["data"])?);
} }
} }
if parsed["data"]["after"].is_string() { if parsed["data"]["after"].is_string() {