added full request and tests

Cargo.lock

@@ -17,6 +17,15 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.98"
@@ -84,6 +93,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
 dependencies = [
  "memchr",
+ "regex-automata",
  "serde",
 ]
 
@@ -151,6 +161,21 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
 [[package]]
 name = "cssparser"
 version = "0.27.2"
@@ -329,6 +354,20 @@ dependencies = [
  "new_debug_unreachable",
 ]
 
+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.31"
@@ -351,6 +390,17 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.31"
@@ -369,8 +419,10 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
+ "futures-channel",
  "futures-core",
  "futures-io",
+ "futures-macro",
  "futures-sink",
  "futures-task",
  "memchr",
@@ -507,6 +559,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "html5ever 0.35.0",
+ "httptest",
  "kuchikiki",
  "markup5ever 0.11.0",
  "mlua",
@@ -562,6 +615,36 @@ version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
 
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
+[[package]]
+name = "httptest"
+version = "0.16.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bde82de3ef9bd882493c6a5edbc3363ad928925b30ccecc0f2ddeb42601b3021"
+dependencies = [
+ "bstr",
+ "bytes",
+ "crossbeam-channel",
+ "form_urlencoded",
+ "futures",
+ "http",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "log",
+ "once_cell",
+ "regex",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+]
+
 [[package]]
 name = "hyper"
 version = "1.6.0"
@@ -575,6 +658,7 @@ dependencies = [
  "http",
  "http-body",
  "httparse",
+ "httpdate",
  "itoa 1.0.15",
  "pin-project-lite",
  "smallvec",
@@ -1502,6 +1586,29 @@ dependencies = [
  "bitflags 2.9.1",
 ]
 
+[[package]]
+name = "regex"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
 [[package]]
 name = "regex-syntax"
 version = "0.8.5"
@@ -2047,9 +2154,21 @@ dependencies = [
  "pin-project-lite",
  "slab",
  "socket2",
+ "tokio-macros",
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "tokio-macros"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
 [[package]]
 name = "tokio-native-tls"
 version = "0.3.1"

htmlua-parser/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.98"
 html5ever = "0.35.0"
+httptest = "0.16.3"
 kuchikiki = "0.8.2"
 markup5ever = "0.11.0"
 mlua = { version = "0.10.5", features = ["lua54", "vendored", "serialize", "anyhow"] }

htmlua-parser/src/helpers.rs

@@ -1,4 +1,8 @@
-use std::{fs::File, io::{BufRead, BufReader, Read, Seek, SeekFrom}, path::PathBuf};
+use std::{
+    fs::File,
+    io::{BufRead, BufReader, Read, Seek, SeekFrom},
+    path::PathBuf,
+};
 
 use anyhow::Result;
 use kuchikiki::NodeRef;
@@ -9,7 +13,6 @@ use tendril::TendrilSink;
 
 
 pub fn read_doc_from_file(path: PathBuf) -> Result<NodeRef> {
-
     let mut file = File::open(path)?;
     let mut reader = BufReader::new(&mut file);
 

htmlua-parser/src/htmlua_stdlib.rs

@@ -1,10 +1,19 @@
-use std::{cell::{LazyCell, RefCell}, collections::HashMap, fmt::Write, rc::Rc};
+use std::{
+    cell::{LazyCell, RefCell},
+    collections::HashMap,
+    fmt::Write,
+    rc::Rc,
+    str::FromStr,
+    time::Duration,
+};
 
-use anyhow::anyhow;
+use mlua::{Error, Lua, Table, prelude::*};
-use mlua::{prelude::*, Error, Lua, Table};
+use reqwest::{
-use reqwest::{blocking::{Client, Response}, header::HeaderMap};
+    Method, Url,
+    blocking::{Client, Response},
+    header::HeaderMap,
+};
 use serde::{Deserialize, Serialize};
-use serde_json::to_value;
 
 
 pub fn create_htmlua_stdlib(l: &Lua, stdout: &Rc<RefCell<String>>) -> mlua::Result<Table> {
@@ -34,13 +43,16 @@ pub fn create_htmlua_stdlib(l: &Lua, stdout: &Rc<RefCell<String>>) -> mlua::Resu
     Ok(t)
 }
 
+#[allow(clippy::too_many_lines)]
 fn create_http_lib(l: &Lua) -> mlua::Result<Table> {
     let t = l.create_table()?;
-    let http_client: Rc<LazyCell<Client>> = Rc::new(LazyCell::new(|| Client::builder()
+    let http_client: Rc<LazyCell<Client>> = Rc::new(LazyCell::new(|| {
+        Client::builder()
             .timeout(std::time::Duration::from_secs(3))
             .user_agent("htmlua/0.1.0")
-            .build().unwrap()
+            .build()
-        ));
+            .unwrap()
+    }));
 
 
     let client = http_client.clone();
@@ -50,56 +62,116 @@ fn create_http_lib(l: &Lua) -> mlua::Result<Table> {
             let res = client.get(url).send().map_err(|e| Error::RuntimeError(e.to_string()))?;
             let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
             Ok(lua_res)
-        })?
+        })?,
     )?;
 
     let client = http_client.clone();
     t.set(
         "post",
         l.create_function(move |_, url: String| {
-            let res = client.post(url).send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let res = client
+                .post(url)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
             let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
             Ok(lua_res)
-        })?
+        })?,
     )?;
 
     let client = http_client.clone();
     t.set(
         "get_with_data",
         l.create_function(move |_, (url, data): (String, HashMap<String, String>)| {
-            let res = client.get(url).query(&data).send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let res = client
+                .get(url)
+                .query(&data)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
             let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
             Ok(lua_res)
-        })?
+        })?,
     )?;
 
     let client = http_client.clone();
     t.set(
         "post_with_data_form",
         l.create_function(move |_, (url, data): (String, HashMap<String, String>)| {
-            let res = client.post(url).form(&data).send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let res = client
+                .post(url)
+                .form(&data)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
             let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
             Ok(lua_res)
-        })?
+        })?,
     )?;
 
     let client = http_client.clone();
     t.set(
         "post_with_data_json",
         l.create_function(move |_, (url, data): (String, HashMap<String, String>)| {
-            let res = client.post(url).json(&data).send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let res = client
+                .post(url)
+                .json(&data)
+                .send()
+                .map_err(|e| Error::RuntimeError(e.to_string()))?;
             let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
             Ok(lua_res)
-        })?
+        })?,
+    )?;
+
+    let client = http_client.clone();
+    t.set(
+        "request",
+        l.create_function(move |_, table: mlua::Table| {
+            let mut request = client.request(
+                Method::from_bytes(table.get::<String>("method")?.as_bytes())
+                    .map_err(|e| Error::RuntimeError(e.to_string()))?,
+                Url::from_str(table.get::<String>("url")?.as_str()).map_err(|e| Error::RuntimeError(e.to_string()))?,
+            );
+
+            if let Ok(header_tbl) = table.get::<mlua::Table>("headers") {
+                request = header_tbl
+                    .pairs::<String, String>()
+                    .filter_map(std::result::Result::ok)
+                    .fold(request, |req, (k, v)| req.header(k, v));
+            }
+
+            if let Ok(basic_auth) = table.get::<mlua::Table>("basic_auth") {
+                request = request
+                    .basic_auth(basic_auth.get::<String>("username")?, basic_auth.get::<String>("password").ok());
+            }
+
+            if let Ok(bearer_auth) = table.get::<mlua::Table>("bearer_auth") {
+                request = request.bearer_auth(bearer_auth.get::<String>("token")?);
+            }
+
+            if let Ok(body) = table.get::<String>("body") {
+                request = request.body(body);
+            }
+
+            if let Ok(json) = table.get::<String>("json") {
+                request = request.json(&json);
+            }
+
+            if let Ok(timeout) = table.get::<u64>("timeout") {
+                request = request.timeout(Duration::from_secs(timeout));
+            }
+
+
+            let res = request.send().map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let lua_res: LuaHttpResponse = TryFrom::try_from(res)?;
+            Ok(lua_res)
+        })?,
     )?;
-    //TODO: all http
 
     t.set(
         "decode_json",
         l.create_function(move |l, text: String| {
-            let table: serde_json::Value = serde_json::from_str(&text).map_err(|e| Error::RuntimeError(e.to_string()))?;
+            let table: serde_json::Value =
+                serde_json::from_str(&text).map_err(|e| Error::RuntimeError(e.to_string()))?;
             Ok(l.to_value(&table))
-        })?
+        })?,
     )?;
 
     Ok(t)
@@ -115,7 +187,11 @@ struct LuaHttpResponse {
 
 impl TryFrom<Response> for LuaHttpResponse {
     fn try_from(value: Response) -> Result<Self, Self::Error> {
-        Ok(LuaHttpResponse { headers: headermap_to_hashmap(value.headers()), status: value.status().as_u16(), body: value.text().map_err(|e| Error::RuntimeError(e.to_string()))?, })
+        Ok(LuaHttpResponse {
+            headers: headermap_to_hashmap(value.headers()),
+            status: value.status().as_u16(),
+            body: value.text().map_err(|e| Error::RuntimeError(e.to_string()))?,
+        })
     }
 
     type Error = Error;
@@ -133,6 +209,7 @@ fn headermap_to_hashmap(headers: &HeaderMap) -> HashMap<String, String> {
     map
 }
 
+
 impl IntoLua for LuaHttpResponse {
     fn into_lua(self, lua: &Lua) -> LuaResult<LuaValue> {
         let table = lua.create_table()?;

htmlua-parser/src/lib.rs

@@ -1,4 +1,4 @@
-pub mod render;
-pub mod serve;
 pub mod helpers;
 pub mod htmlua_stdlib;
+pub mod render;
+pub mod serve;

htmlua-parser/src/render.rs

@@ -1,8 +1,13 @@
-use std::{cell::RefCell, fmt::Write, fs::read_to_string, path::PathBuf, rc::Rc};
+use std::{
+    cell::{LazyCell, RefCell},
+    fmt::Write,
+    path::PathBuf,
+    rc::Rc,
+};
 
 use anyhow::{Result, anyhow};
 use kuchikiki::{NodeRef, traits::TendrilSink};
-use mlua::{Lua, Table};
+use mlua::Lua;
 use pulldown_cmark::{Options, Parser, html};
 use syntect::{
     easy::HighlightLines,
@@ -15,20 +20,22 @@ use syntect::{
 use crate::{helpers::read_doc_from_file, htmlua_stdlib::create_htmlua_stdlib};
 
 
-
+fn build_lua_with_stdout(stdout: &Rc<RefCell<String>>) -> Result<Lua> {
-pub fn execute_lua(document: NodeRef) -> Result<NodeRef> {
     let lua = Lua::new();
     let globals = lua.globals();
 
-    let stdout: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new()));
+    let htmlua_table = create_htmlua_stdlib(&lua, stdout).map_err(|e| anyhow!("Failed to create Lua stdlib: {}", e))?;
-
-    let htmlua_table =
-        create_htmlua_stdlib(&lua, &stdout).map_err(|e| anyhow!("Failed to create Lua stdlib: {}", e))?;
 
     globals
         .set("htmlua", htmlua_table)
         .map_err(|e| anyhow!("Failed to set global: {}", e))?;
 
+    Ok(lua)
+}
+
+pub fn execute_lua(document: NodeRef) -> Result<NodeRef> {
+    let stdout: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new()));
+    let lua = LazyCell::new(|| build_lua_with_stdout(&stdout).unwrap_or_default());
     let lua_elements: Vec<_> = match document.select("lua") {
         Ok(e) => e.collect(),
         Err(()) => return Err(anyhow!("Unable to find Lua")),
@@ -85,7 +92,11 @@ pub fn expand_template(document: NodeRef, component_path: &PathBuf, include_from
                 let exported = from_node
                     .select_first(format!("exportelement.{name}").as_str())
                     .map_err(|()| anyhow!("Error finding exportelement"))?;
-                exported.as_node().children().rev().for_each(|c| i.as_node().insert_after(c));
+                exported
+                    .as_node()
+                    .children()
+                    .rev()
+                    .for_each(|c| i.as_node().insert_after(c));
             }
         }
         while let Ok(i) = document.select_first("includeelement") {
@@ -107,8 +118,13 @@ pub fn expand_template(document: NodeRef, component_path: &PathBuf, include_from
             item_path.push(include_path);
             let new_node = read_doc_from_file(item_path)?;
             let replaced_node = expand_template(new_node, component_path, Some(i.as_node()))?;
-            replaced_node.select_first("html").map_err(|()| anyhow!("Error finding html"))?.as_node().children().rev().for_each(|c| i.as_node().insert_after(c));
+            replaced_node
-
+                .select_first("html")
+                .map_err(|()| anyhow!("Error finding html"))?
+                .as_node()
+                .children()
+                .rev()
+                .for_each(|c| i.as_node().insert_after(c));
         }
     }
     while let Ok(i) = document.select_first("include") {
@@ -167,6 +183,8 @@ pub fn process_syntax_highlighting(document: NodeRef) -> Result<NodeRef> {
 
 #[cfg(test)]
 mod tests {
+    use httptest::{Expectation, ServerPool, matchers::*, responders::*};
+
     use super::*;
 
     #[test]
@@ -413,4 +431,141 @@ fn main() {
         assert_eq!(attrs.get("data-lang"), Some("rust"));
         assert!(attrs.get("class").unwrap().contains("syntax-highlight"));
     }
+
+    static SERVER_POOL: ServerPool = ServerPool::new(2);
+
+    #[test]
+    fn basic_get() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(request::method_path("GET", "/test/1")).respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!("htmlua.print(htmlua.http.get(\"{}\").body)", server.url("/test/1"));
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn basic_post() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(request::method_path("POST", "/test/1")).respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!("htmlua.print(htmlua.http.post(\"{}\").body)", server.url("/test/1"));
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn get_with_data() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(request::query(url_decoded(contains(("a", "b")))))
+                .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                data = {{}}
+                data.a = \"b\"
+                htmlua.print(htmlua.http.get_with_data(\"{}\", data).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn post_with_data_form() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(all_of![request::method_path("POST", "/test/1"), request::body("a=b")])
+                .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                data = {{}}
+                data.a = \"b\"
+                htmlua.print(htmlua.http.post_with_data_form(\"{}\", data).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn post_with_data_json() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(all_of![request::method_path("POST", "/test/1"), request::body(r#"{"a":"b"}"#)])
+                .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                data = {{}}
+                data.a = \"b\"
+                htmlua.print(htmlua.http.post_with_data_json(\"{}\", data).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
+
+    #[test]
+    fn table_request() {
+        let server = SERVER_POOL.get_server();
+
+        server.expect(
+            Expectation::matching(all_of![
+                request::method_path("GET", "/test/1"),
+                request::body("cool"),
+                request::headers(contains(("testhdr", "val"))),
+                request::headers(contains(("authorization", "Basic dXNlcjpwYXNz"))),
+            ])
+            .respond_with(status_code(200).body("ret")),
+        );
+
+        let stdout = Rc::new(RefCell::new(String::new()));
+        let lua = build_lua_with_stdout(&stdout).unwrap();
+        let code = format!(
+            "
+                req = {{}}
+                req.url = \"{}\"
+                req.method = \"GET\"
+                req.headers = {{}}
+                req.headers.testhdr = \"val\"
+                req.basic_auth = {{}}
+                req.basic_auth.username = \"user\"
+                req.basic_auth.password = \"pass\"
+                req.body = \"cool\"
+
+                htmlua.print(htmlua.http.request(req).body)
+            ",
+            server.url("/test/1")
+        );
+        lua.load(code).exec().unwrap();
+        assert_eq!(stdout.borrow().as_str(), "ret");
+    }
 }

htmlua-parser/src/serve.rs

@@ -2,8 +2,7 @@ use crate::helpers::read_doc_from_file;
 
 use super::render::{execute_lua, expand_template, process_markdown, process_syntax_highlighting};
 use anyhow::Result;
-use std::path::{Path, PathBuf}
+use std::path::{Path, PathBuf};
-;
 
 pub fn serve_content(request_uri: &str) -> Result<String> {
     // TODO: read from config