Commit ecfa39ac23e21b5d630bc689747747f503f942ef
1 parent
1667225a
improbe config and upload informations handling
Showing
9 changed files
with
151 additions
and
61 deletions
artshop.toml
0 → 100644
@@ -22,8 +22,10 @@ flate2 = "^1.0" | @@ -22,8 +22,10 @@ flate2 = "^1.0" | ||
22 | futures = "^0.3" | 22 | futures = "^0.3" |
23 | futures-util = { version = "0", features = ["std"] } | 23 | futures-util = { version = "0", features = ["std"] } |
24 | listenfd = "0.3" | 24 | listenfd = "0.3" |
25 | +once_cell = "^1.9" | ||
25 | r2d2 = "0.8.9" | 26 | r2d2 = "0.8.9" |
26 | serde = { version = "^1.0", features = ["derive"] } | 27 | serde = { version = "^1.0", features = ["derive"] } |
27 | serde_derive = "1.0" | 28 | serde_derive = "1.0" |
28 | serde_json = "1.0" | 29 | serde_json = "1.0" |
30 | +toml = "^0.5" | ||
29 | uuid = { version = "^0.8", features = ["v4", "v5"] } | 31 | uuid = { version = "^0.8", features = ["v4", "v5"] } |
server/src/config.rs
0 → 100644
1 | +use std::fs::File; | ||
2 | +use std::io::Read; | ||
3 | +use once_cell::sync::Lazy; | ||
4 | +use serde::Deserialize; | ||
5 | + | ||
6 | +#[derive(Debug, Deserialize)] | ||
7 | +struct Database { url :Option<String> } | ||
8 | + | ||
9 | +#[derive(Debug, Deserialize)] | ||
10 | +struct Locations { upload :String | ||
11 | + , images :String } | ||
12 | + | ||
13 | +#[derive(Debug, Deserialize)] | ||
14 | +pub(crate) struct Config { database :Database | ||
15 | + , locations :Locations } | ||
16 | + | ||
17 | +pub(crate) static CONFIG :Lazy<Config> = Lazy::new(|| Config::load()); | ||
18 | + | ||
19 | +impl Config { | ||
20 | + pub fn load() -> Config { | ||
21 | + let filename = std::env::var("CONFIG").unwrap(); | ||
22 | + | ||
23 | + let mut buffer = vec![]; | ||
24 | + let mut file = File::open(filename).unwrap(); | ||
25 | + | ||
26 | + file.read_to_end(&mut buffer).unwrap(); | ||
27 | + let mut config :Config = toml::from_slice(&buffer).unwrap(); | ||
28 | + | ||
29 | + config.database.url = match config.database.url { | ||
30 | + Some(url) => Some(url), | ||
31 | + None => std::env::var("DATABASE_URL").ok() | ||
32 | + }; | ||
33 | + | ||
34 | + config | ||
35 | + } | ||
36 | + | ||
37 | + pub fn upload_dir(&self) -> &str { | ||
38 | + self.locations.upload.as_str() | ||
39 | + } | ||
40 | + | ||
41 | + pub fn images_dir(&self) -> &str { | ||
42 | + self.locations.images.as_str() | ||
43 | + } | ||
44 | +} |
1 | #[macro_use] | 1 | #[macro_use] |
2 | extern crate diesel; | 2 | extern crate diesel; |
3 | 3 | ||
4 | +mod config; | ||
4 | mod error; | 5 | mod error; |
5 | mod models; | 6 | mod models; |
6 | mod routes; | 7 | mod routes; |
7 | mod schema; | 8 | mod schema; |
8 | mod uuid; | 9 | mod uuid; |
9 | -mod upload; | 10 | +mod upload_worker; |
10 | 11 | ||
11 | -use async_std::channel::Receiver; | ||
12 | use models::image::Image; | 12 | use models::image::Image; |
13 | use routes::markdown::*; | 13 | use routes::markdown::*; |
14 | use routes::other::*; | 14 | use routes::other::*; |
15 | use routes::user::*; | 15 | use routes::user::*; |
16 | use routes::upload::*; | 16 | use routes::upload::*; |
17 | -use crate::upload::get_sample; | ||
18 | -use crate::uuid::Uuid; | ||
19 | 17 | ||
20 | use actix_web::{guard, web, App, HttpResponse, HttpServer}; | 18 | use actix_web::{guard, web, App, HttpResponse, HttpServer}; |
21 | -use async_std::{ channel::{ Sender, bounded } | ||
22 | - , fs::File }; | 19 | +use async_std::channel::Sender; |
23 | use diesel::r2d2::{self, ConnectionManager}; | 20 | use diesel::r2d2::{self, ConnectionManager}; |
24 | use diesel::SqliteConnection; | 21 | use diesel::SqliteConnection; |
25 | -use futures::{ FutureExt, StreamExt, select | ||
26 | - , stream::FuturesUnordered }; | ||
27 | use listenfd::ListenFd; | 22 | use listenfd::ListenFd; |
28 | -use std::convert::TryFrom; | ||
29 | use std::sync::Arc; | 23 | use std::sync::Arc; |
24 | +use std::ops::Deref; | ||
30 | 25 | ||
31 | pub(crate) type Pool = r2d2::Pool<ConnectionManager<SqliteConnection>>; | 26 | pub(crate) type Pool = r2d2::Pool<ConnectionManager<SqliteConnection>>; |
32 | 27 | ||
@@ -42,39 +37,9 @@ async fn main() -> std::io::Result<()> { | @@ -42,39 +37,9 @@ async fn main() -> std::io::Result<()> { | ||
42 | 37 | ||
43 | dotenv::dotenv().ok(); | 38 | dotenv::dotenv().ok(); |
44 | 39 | ||
45 | - let (tx_upload_worker, rx_upload_worker) | ||
46 | - : (Sender<Image>, Receiver<Image>) = bounded(32); | 40 | + println!("CONFIG: {:?}", config::CONFIG.deref()); |
47 | 41 | ||
48 | - let _upload_worker = actix_rt::spawn(async move { | ||
49 | - let mut workers = FuturesUnordered::new(); | ||
50 | - | ||
51 | - loop { | ||
52 | - select! { | ||
53 | - filename = rx_upload_worker.recv().fuse() => { | ||
54 | - match filename { | ||
55 | - Err(_) => break, | ||
56 | - Ok(upload) => workers.push(async move { | ||
57 | - let upload_uuid = Uuid::try_from(upload.upload_uuid.unwrap()).unwrap(); | ||
58 | - let upload_filename = format!("/tmp/upload_{}", upload_uuid); | ||
59 | - let mut f = File::open(&upload_filename).await.unwrap(); | ||
60 | - let mut buf = vec!['.' as u8; 3 * 4096]; | ||
61 | - get_sample(&mut f, buf.as_mut()).await.unwrap(); | ||
62 | - println!( "[UPLOAD WORKER] filename: {}" | ||
63 | - , upload_filename ); | ||
64 | - println!( "[UPLOAD WORKER] uuid: {}" | ||
65 | - , Uuid::get( "some.unique.namespace" | ||
66 | - , buf.as_mut() ) ); | ||
67 | - }) | ||
68 | - } | ||
69 | - }, | ||
70 | - _result = workers.next() => {}, | ||
71 | - } | ||
72 | - } | ||
73 | - | ||
74 | - while workers.len() > 0 { | ||
75 | - workers.next().await; | ||
76 | - } | ||
77 | - }); | 42 | + let tx_upload_worker = upload_worker::launch(); |
78 | 43 | ||
79 | let database_url = std::env::var("DATABASE_URL").expect("NOT FOUND"); | 44 | let database_url = std::env::var("DATABASE_URL").expect("NOT FOUND"); |
80 | let database_pool = Pool::builder() | 45 | let database_pool = Pool::builder() |
@@ -37,6 +37,27 @@ pub struct Upload { | @@ -37,6 +37,27 @@ pub struct Upload { | ||
37 | pub mime_type :String, | 37 | pub mime_type :String, |
38 | } | 38 | } |
39 | 39 | ||
40 | +#[macro_export] | ||
41 | +macro_rules! upload_uuid { | ||
42 | + ($u:expr) => { | ||
43 | + match &$u.upload_uuid { | ||
44 | + Some(uuid) => Uuid::try_from(uuid.as_slice()).ok(), | ||
45 | + None => None, | ||
46 | + } | ||
47 | + }; | ||
48 | +} | ||
49 | + | ||
50 | +#[macro_export] | ||
51 | +macro_rules! upload_filename { | ||
52 | + ($u:expr) => { | ||
53 | + upload_uuid!($u).and_then(|uuid| | ||
54 | + Some(format!( "{}/upload_{}" | ||
55 | + , CONFIG.upload_dir() | ||
56 | + , uuid))) | ||
57 | + }; | ||
58 | +} | ||
59 | + | ||
60 | + | ||
40 | pub(crate) fn upload( pool: Arc<Pool> | 61 | pub(crate) fn upload( pool: Arc<Pool> |
41 | , item: Upload ) -> Result<Image> { | 62 | , item: Upload ) -> Result<Image> { |
42 | use crate::schema::images::dsl::*; | 63 | use crate::schema::images::dsl::*; |
1 | use actix_web::{Error, HttpResponse, web}; | 1 | use actix_web::{Error, HttpResponse, web}; |
2 | use anyhow::Result; | 2 | use anyhow::Result; |
3 | +use async_std::fs::DirBuilder; | ||
3 | use futures::stream::StreamExt; | 4 | use futures::stream::StreamExt; |
4 | use async_std::{fs::OpenOptions, io::WriteExt}; | 5 | use async_std::{fs::OpenOptions, io::WriteExt}; |
5 | -use uuid::Uuid; | 6 | +use crate::uuid::Uuid; |
6 | 7 | ||
7 | -use crate::{AppData, models::image::{Upload, self}}; | 8 | +use crate::{AppData, models::image::{Upload, self}, upload_filename, upload_uuid}; |
9 | +use crate::config::CONFIG; | ||
10 | +use std::convert::TryFrom; | ||
8 | 11 | ||
9 | pub async fn upload( app_data :web::Data<AppData> | 12 | pub async fn upload( app_data :web::Data<AppData> |
10 | , mut body :web::Payload | 13 | , mut body :web::Payload |
@@ -13,8 +16,7 @@ pub async fn upload( app_data :web::Data<AppData> | @@ -13,8 +16,7 @@ pub async fn upload( app_data :web::Data<AppData> | ||
13 | let pool = app_data.database_pool.clone(); | 16 | let pool = app_data.database_pool.clone(); |
14 | let worker = app_data.tx_upload_worker.clone(); | 17 | let worker = app_data.tx_upload_worker.clone(); |
15 | 18 | ||
16 | - let random_uuid = Uuid::new_v4(); | ||
17 | - let upload_uuid = Some(random_uuid.as_bytes().to_vec()); | 19 | + let upload_uuid = Some(uuid::Uuid::new_v4().as_bytes().to_vec()); |
18 | let size = request.headers().get("content-length") | 20 | let size = request.headers().get("content-length") |
19 | . and_then(|h| Some(h.to_str().unwrap().parse::<i32>())) | 21 | . and_then(|h| Some(h.to_str().unwrap().parse::<i32>())) |
20 | . unwrap().unwrap(); | 22 | . unwrap().unwrap(); |
@@ -22,23 +24,26 @@ pub async fn upload( app_data :web::Data<AppData> | @@ -22,23 +24,26 @@ pub async fn upload( app_data :web::Data<AppData> | ||
22 | . and_then(|h| Some(h.to_str().unwrap())) | 24 | . and_then(|h| Some(h.to_str().unwrap())) |
23 | . unwrap() ); | 25 | . unwrap() ); |
24 | 26 | ||
25 | - let upload_filename = format!("/tmp/upload_{}", random_uuid); | 27 | + let upload = Upload { |
28 | + upload_uuid, | ||
29 | + size, | ||
30 | + mime_type | ||
31 | + }; | ||
32 | + | ||
33 | + DirBuilder::new() . recursive(true) | ||
34 | + . create(CONFIG.upload_dir()) | ||
35 | + . await?; | ||
36 | + | ||
37 | + let upload_filename = upload_filename!(upload).unwrap(); | ||
26 | let mut output = OpenOptions::new(); | 38 | let mut output = OpenOptions::new(); |
27 | let mut output = output | 39 | let mut output = output |
28 | . create(true) | 40 | . create(true) |
29 | . write(true) | 41 | . write(true) |
30 | . open(&upload_filename).await?; | 42 | . open(&upload_filename).await?; |
31 | - | ||
32 | while let Some(item) = body.next().await { | 43 | while let Some(item) = body.next().await { |
33 | output.write_all(&item?).await?; | 44 | output.write_all(&item?).await?; |
34 | } | 45 | } |
35 | 46 | ||
36 | - let upload = Upload { | ||
37 | - upload_uuid, | ||
38 | - size, | ||
39 | - mime_type | ||
40 | - }; | ||
41 | - | ||
42 | Ok( match web::block(move || image::upload(pool, upload)).await { | 47 | Ok( match web::block(move || image::upload(pool, upload)).await { |
43 | Ok(image) => { | 48 | Ok(image) => { |
44 | // TODO handle this as error response... | 49 | // TODO handle this as error response... |
1 | use std::io::SeekFrom; | 1 | use std::io::SeekFrom; |
2 | -use async_std::fs::File; | ||
3 | -use futures::{AsyncSeekExt, AsyncReadExt}; | 2 | +use async_std::{fs::File, channel::{Sender, Receiver, bounded}}; |
3 | +use futures::{ AsyncSeekExt, AsyncReadExt, FutureExt, StreamExt, select | ||
4 | + , stream::FuturesUnordered}; | ||
4 | 5 | ||
6 | +use crate::{models::image::Image, upload_filename, upload_uuid}; | ||
7 | +use crate::uuid::Uuid; | ||
8 | + | ||
9 | +use std::convert::TryFrom; | ||
10 | +use crate::config::CONFIG; | ||
11 | + | ||
12 | +pub fn launch() -> Sender<Image> { | ||
13 | + let (tx_upload_worker, rx_upload_worker) | ||
14 | + : (Sender<Image>, Receiver<Image>) = bounded(32); | ||
15 | + | ||
16 | + actix_rt::spawn(async move { | ||
17 | + let mut workers = FuturesUnordered::new(); | ||
18 | + | ||
19 | + loop { | ||
20 | + select! { | ||
21 | + image = rx_upload_worker.recv().fuse() => { | ||
22 | + match image { | ||
23 | + Err(_) => break, | ||
24 | + Ok(image) => workers.push(worker(image)), | ||
25 | + } | ||
26 | + }, | ||
27 | + _result = workers.next() => {}, | ||
28 | + } | ||
29 | + } | ||
30 | + | ||
31 | + while workers.len() > 0 { | ||
32 | + workers.next().await; | ||
33 | + } | ||
34 | + }); | ||
35 | + | ||
36 | + tx_upload_worker | ||
37 | +} | ||
38 | + | ||
39 | + | ||
40 | +async fn worker(image :Image) { | ||
41 | + let upload_filename = upload_filename!(image).unwrap(); | ||
42 | + let mut f = File::open(&upload_filename).await.unwrap(); | ||
43 | + let mut buf = vec!['.' as u8; 3 * 4096]; | ||
44 | + get_sample(&mut f, buf.as_mut()).await.unwrap(); | ||
45 | + println!( "[upload worker] filename: {}" | ||
46 | + , upload_filename ); | ||
47 | + println!( "[upload worker] uuid: {}" | ||
48 | + , Uuid::get( "some.unique.namespace" | ||
49 | + , buf.as_mut() ) ); | ||
50 | +} | ||
5 | 51 | ||
6 | async fn read_at( f :&mut File | 52 | async fn read_at( f :&mut File |
7 | - , pos :SeekFrom | ||
8 | - , buf :&mut [u8]) -> std::io::Result<()> { | 53 | + , pos :SeekFrom |
54 | + , buf :&mut [u8]) -> std::io::Result<()> { | ||
9 | f.seek(pos).await?; | 55 | f.seek(pos).await?; |
10 | f.read_exact(buf).await | 56 | f.read_exact(buf).await |
11 | } | 57 | } |
12 | 58 | ||
13 | -pub async fn get_sample( f :&mut File | 59 | +async fn get_sample( f :&mut File |
14 | , buf :&mut [u8]) -> std::io::Result<()> { | 60 | , buf :&mut [u8]) -> std::io::Result<()> { |
15 | let file_len = f.metadata().await?.len(); | 61 | let file_len = f.metadata().await?.len(); |
16 | let chunk_size = buf.len() / 3; | 62 | let chunk_size = buf.len() / 3; |
@@ -23,10 +23,10 @@ impl Uuid { | @@ -23,10 +23,10 @@ impl Uuid { | ||
23 | } | 23 | } |
24 | } | 24 | } |
25 | 25 | ||
26 | -impl TryFrom<Vec<u8>> for Uuid { | 26 | +impl TryFrom<&[u8]> for Uuid { |
27 | type Error = Error; | 27 | type Error = Error; |
28 | 28 | ||
29 | - fn try_from(value: Vec<u8>) -> Result<Self, Self::Error> { | ||
30 | - Ok(Self(uuid::Uuid::from_slice(value.as_slice())?)) | 29 | + fn try_from(value: &[u8]) -> Result<Self, Self::Error> { |
30 | + Ok(Self(uuid::Uuid::from_slice(value)?)) | ||
31 | } | 31 | } |
32 | } | 32 | } |
Please
register
or
login
to post a comment