Commit ecfa39ac23e21b5d630bc689747747f503f942ef
1 parent
1667225a
improbe config and upload informations handling
Showing
9 changed files
with
151 additions
and
61 deletions
artshop.toml
0 → 100644
| ... | ... | @@ -22,8 +22,10 @@ flate2 = "^1.0" |
| 22 | 22 | futures = "^0.3" |
| 23 | 23 | futures-util = { version = "0", features = ["std"] } |
| 24 | 24 | listenfd = "0.3" |
| 25 | +once_cell = "^1.9" | |
| 25 | 26 | r2d2 = "0.8.9" |
| 26 | 27 | serde = { version = "^1.0", features = ["derive"] } |
| 27 | 28 | serde_derive = "1.0" |
| 28 | 29 | serde_json = "1.0" |
| 30 | +toml = "^0.5" | |
| 29 | 31 | uuid = { version = "^0.8", features = ["v4", "v5"] } | ... | ... |
server/src/config.rs
0 → 100644
| 1 | +use std::fs::File; | |
| 2 | +use std::io::Read; | |
| 3 | +use once_cell::sync::Lazy; | |
| 4 | +use serde::Deserialize; | |
| 5 | + | |
| 6 | +#[derive(Debug, Deserialize)] | |
| 7 | +struct Database { url :Option<String> } | |
| 8 | + | |
| 9 | +#[derive(Debug, Deserialize)] | |
| 10 | +struct Locations { upload :String | |
| 11 | + , images :String } | |
| 12 | + | |
| 13 | +#[derive(Debug, Deserialize)] | |
| 14 | +pub(crate) struct Config { database :Database | |
| 15 | + , locations :Locations } | |
| 16 | + | |
| 17 | +pub(crate) static CONFIG :Lazy<Config> = Lazy::new(|| Config::load()); | |
| 18 | + | |
| 19 | +impl Config { | |
| 20 | + pub fn load() -> Config { | |
| 21 | + let filename = std::env::var("CONFIG").unwrap(); | |
| 22 | + | |
| 23 | + let mut buffer = vec![]; | |
| 24 | + let mut file = File::open(filename).unwrap(); | |
| 25 | + | |
| 26 | + file.read_to_end(&mut buffer).unwrap(); | |
| 27 | + let mut config :Config = toml::from_slice(&buffer).unwrap(); | |
| 28 | + | |
| 29 | + config.database.url = match config.database.url { | |
| 30 | + Some(url) => Some(url), | |
| 31 | + None => std::env::var("DATABASE_URL").ok() | |
| 32 | + }; | |
| 33 | + | |
| 34 | + config | |
| 35 | + } | |
| 36 | + | |
| 37 | + pub fn upload_dir(&self) -> &str { | |
| 38 | + self.locations.upload.as_str() | |
| 39 | + } | |
| 40 | + | |
| 41 | + pub fn images_dir(&self) -> &str { | |
| 42 | + self.locations.images.as_str() | |
| 43 | + } | |
| 44 | +} | ... | ... |
| 1 | 1 | #[macro_use] |
| 2 | 2 | extern crate diesel; |
| 3 | 3 | |
| 4 | +mod config; | |
| 4 | 5 | mod error; |
| 5 | 6 | mod models; |
| 6 | 7 | mod routes; |
| 7 | 8 | mod schema; |
| 8 | 9 | mod uuid; |
| 9 | -mod upload; | |
| 10 | +mod upload_worker; | |
| 10 | 11 | |
| 11 | -use async_std::channel::Receiver; | |
| 12 | 12 | use models::image::Image; |
| 13 | 13 | use routes::markdown::*; |
| 14 | 14 | use routes::other::*; |
| 15 | 15 | use routes::user::*; |
| 16 | 16 | use routes::upload::*; |
| 17 | -use crate::upload::get_sample; | |
| 18 | -use crate::uuid::Uuid; | |
| 19 | 17 | |
| 20 | 18 | use actix_web::{guard, web, App, HttpResponse, HttpServer}; |
| 21 | -use async_std::{ channel::{ Sender, bounded } | |
| 22 | - , fs::File }; | |
| 19 | +use async_std::channel::Sender; | |
| 23 | 20 | use diesel::r2d2::{self, ConnectionManager}; |
| 24 | 21 | use diesel::SqliteConnection; |
| 25 | -use futures::{ FutureExt, StreamExt, select | |
| 26 | - , stream::FuturesUnordered }; | |
| 27 | 22 | use listenfd::ListenFd; |
| 28 | -use std::convert::TryFrom; | |
| 29 | 23 | use std::sync::Arc; |
| 24 | +use std::ops::Deref; | |
| 30 | 25 | |
| 31 | 26 | pub(crate) type Pool = r2d2::Pool<ConnectionManager<SqliteConnection>>; |
| 32 | 27 | |
| ... | ... | @@ -42,39 +37,9 @@ async fn main() -> std::io::Result<()> { |
| 42 | 37 | |
| 43 | 38 | dotenv::dotenv().ok(); |
| 44 | 39 | |
| 45 | - let (tx_upload_worker, rx_upload_worker) | |
| 46 | - : (Sender<Image>, Receiver<Image>) = bounded(32); | |
| 40 | + println!("CONFIG: {:?}", config::CONFIG.deref()); | |
| 47 | 41 | |
| 48 | - let _upload_worker = actix_rt::spawn(async move { | |
| 49 | - let mut workers = FuturesUnordered::new(); | |
| 50 | - | |
| 51 | - loop { | |
| 52 | - select! { | |
| 53 | - filename = rx_upload_worker.recv().fuse() => { | |
| 54 | - match filename { | |
| 55 | - Err(_) => break, | |
| 56 | - Ok(upload) => workers.push(async move { | |
| 57 | - let upload_uuid = Uuid::try_from(upload.upload_uuid.unwrap()).unwrap(); | |
| 58 | - let upload_filename = format!("/tmp/upload_{}", upload_uuid); | |
| 59 | - let mut f = File::open(&upload_filename).await.unwrap(); | |
| 60 | - let mut buf = vec!['.' as u8; 3 * 4096]; | |
| 61 | - get_sample(&mut f, buf.as_mut()).await.unwrap(); | |
| 62 | - println!( "[UPLOAD WORKER] filename: {}" | |
| 63 | - , upload_filename ); | |
| 64 | - println!( "[UPLOAD WORKER] uuid: {}" | |
| 65 | - , Uuid::get( "some.unique.namespace" | |
| 66 | - , buf.as_mut() ) ); | |
| 67 | - }) | |
| 68 | - } | |
| 69 | - }, | |
| 70 | - _result = workers.next() => {}, | |
| 71 | - } | |
| 72 | - } | |
| 73 | - | |
| 74 | - while workers.len() > 0 { | |
| 75 | - workers.next().await; | |
| 76 | - } | |
| 77 | - }); | |
| 42 | + let tx_upload_worker = upload_worker::launch(); | |
| 78 | 43 | |
| 79 | 44 | let database_url = std::env::var("DATABASE_URL").expect("NOT FOUND"); |
| 80 | 45 | let database_pool = Pool::builder() | ... | ... |
| ... | ... | @@ -37,6 +37,27 @@ pub struct Upload { |
| 37 | 37 | pub mime_type :String, |
| 38 | 38 | } |
| 39 | 39 | |
| 40 | +#[macro_export] | |
| 41 | +macro_rules! upload_uuid { | |
| 42 | + ($u:expr) => { | |
| 43 | + match &$u.upload_uuid { | |
| 44 | + Some(uuid) => Uuid::try_from(uuid.as_slice()).ok(), | |
| 45 | + None => None, | |
| 46 | + } | |
| 47 | + }; | |
| 48 | +} | |
| 49 | + | |
| 50 | +#[macro_export] | |
| 51 | +macro_rules! upload_filename { | |
| 52 | + ($u:expr) => { | |
| 53 | + upload_uuid!($u).and_then(|uuid| | |
| 54 | + Some(format!( "{}/upload_{}" | |
| 55 | + , CONFIG.upload_dir() | |
| 56 | + , uuid))) | |
| 57 | + }; | |
| 58 | +} | |
| 59 | + | |
| 60 | + | |
| 40 | 61 | pub(crate) fn upload( pool: Arc<Pool> |
| 41 | 62 | , item: Upload ) -> Result<Image> { |
| 42 | 63 | use crate::schema::images::dsl::*; | ... | ... |
| 1 | 1 | use actix_web::{Error, HttpResponse, web}; |
| 2 | 2 | use anyhow::Result; |
| 3 | +use async_std::fs::DirBuilder; | |
| 3 | 4 | use futures::stream::StreamExt; |
| 4 | 5 | use async_std::{fs::OpenOptions, io::WriteExt}; |
| 5 | -use uuid::Uuid; | |
| 6 | +use crate::uuid::Uuid; | |
| 6 | 7 | |
| 7 | -use crate::{AppData, models::image::{Upload, self}}; | |
| 8 | +use crate::{AppData, models::image::{Upload, self}, upload_filename, upload_uuid}; | |
| 9 | +use crate::config::CONFIG; | |
| 10 | +use std::convert::TryFrom; | |
| 8 | 11 | |
| 9 | 12 | pub async fn upload( app_data :web::Data<AppData> |
| 10 | 13 | , mut body :web::Payload |
| ... | ... | @@ -13,8 +16,7 @@ pub async fn upload( app_data :web::Data<AppData> |
| 13 | 16 | let pool = app_data.database_pool.clone(); |
| 14 | 17 | let worker = app_data.tx_upload_worker.clone(); |
| 15 | 18 | |
| 16 | - let random_uuid = Uuid::new_v4(); | |
| 17 | - let upload_uuid = Some(random_uuid.as_bytes().to_vec()); | |
| 19 | + let upload_uuid = Some(uuid::Uuid::new_v4().as_bytes().to_vec()); | |
| 18 | 20 | let size = request.headers().get("content-length") |
| 19 | 21 | . and_then(|h| Some(h.to_str().unwrap().parse::<i32>())) |
| 20 | 22 | . unwrap().unwrap(); |
| ... | ... | @@ -22,23 +24,26 @@ pub async fn upload( app_data :web::Data<AppData> |
| 22 | 24 | . and_then(|h| Some(h.to_str().unwrap())) |
| 23 | 25 | . unwrap() ); |
| 24 | 26 | |
| 25 | - let upload_filename = format!("/tmp/upload_{}", random_uuid); | |
| 27 | + let upload = Upload { | |
| 28 | + upload_uuid, | |
| 29 | + size, | |
| 30 | + mime_type | |
| 31 | + }; | |
| 32 | + | |
| 33 | + DirBuilder::new() . recursive(true) | |
| 34 | + . create(CONFIG.upload_dir()) | |
| 35 | + . await?; | |
| 36 | + | |
| 37 | + let upload_filename = upload_filename!(upload).unwrap(); | |
| 26 | 38 | let mut output = OpenOptions::new(); |
| 27 | 39 | let mut output = output |
| 28 | 40 | . create(true) |
| 29 | 41 | . write(true) |
| 30 | 42 | . open(&upload_filename).await?; |
| 31 | - | |
| 32 | 43 | while let Some(item) = body.next().await { |
| 33 | 44 | output.write_all(&item?).await?; |
| 34 | 45 | } |
| 35 | 46 | |
| 36 | - let upload = Upload { | |
| 37 | - upload_uuid, | |
| 38 | - size, | |
| 39 | - mime_type | |
| 40 | - }; | |
| 41 | - | |
| 42 | 47 | Ok( match web::block(move || image::upload(pool, upload)).await { |
| 43 | 48 | Ok(image) => { |
| 44 | 49 | // TODO handle this as error response... | ... | ... |
| 1 | 1 | use std::io::SeekFrom; |
| 2 | -use async_std::fs::File; | |
| 3 | -use futures::{AsyncSeekExt, AsyncReadExt}; | |
| 2 | +use async_std::{fs::File, channel::{Sender, Receiver, bounded}}; | |
| 3 | +use futures::{ AsyncSeekExt, AsyncReadExt, FutureExt, StreamExt, select | |
| 4 | + , stream::FuturesUnordered}; | |
| 4 | 5 | |
| 6 | +use crate::{models::image::Image, upload_filename, upload_uuid}; | |
| 7 | +use crate::uuid::Uuid; | |
| 8 | + | |
| 9 | +use std::convert::TryFrom; | |
| 10 | +use crate::config::CONFIG; | |
| 11 | + | |
| 12 | +pub fn launch() -> Sender<Image> { | |
| 13 | + let (tx_upload_worker, rx_upload_worker) | |
| 14 | + : (Sender<Image>, Receiver<Image>) = bounded(32); | |
| 15 | + | |
| 16 | + actix_rt::spawn(async move { | |
| 17 | + let mut workers = FuturesUnordered::new(); | |
| 18 | + | |
| 19 | + loop { | |
| 20 | + select! { | |
| 21 | + image = rx_upload_worker.recv().fuse() => { | |
| 22 | + match image { | |
| 23 | + Err(_) => break, | |
| 24 | + Ok(image) => workers.push(worker(image)), | |
| 25 | + } | |
| 26 | + }, | |
| 27 | + _result = workers.next() => {}, | |
| 28 | + } | |
| 29 | + } | |
| 30 | + | |
| 31 | + while workers.len() > 0 { | |
| 32 | + workers.next().await; | |
| 33 | + } | |
| 34 | + }); | |
| 35 | + | |
| 36 | + tx_upload_worker | |
| 37 | +} | |
| 38 | + | |
| 39 | + | |
| 40 | +async fn worker(image :Image) { | |
| 41 | + let upload_filename = upload_filename!(image).unwrap(); | |
| 42 | + let mut f = File::open(&upload_filename).await.unwrap(); | |
| 43 | + let mut buf = vec!['.' as u8; 3 * 4096]; | |
| 44 | + get_sample(&mut f, buf.as_mut()).await.unwrap(); | |
| 45 | + println!( "[upload worker] filename: {}" | |
| 46 | + , upload_filename ); | |
| 47 | + println!( "[upload worker] uuid: {}" | |
| 48 | + , Uuid::get( "some.unique.namespace" | |
| 49 | + , buf.as_mut() ) ); | |
| 50 | +} | |
| 5 | 51 | |
| 6 | 52 | async fn read_at( f :&mut File |
| 7 | - , pos :SeekFrom | |
| 8 | - , buf :&mut [u8]) -> std::io::Result<()> { | |
| 53 | + , pos :SeekFrom | |
| 54 | + , buf :&mut [u8]) -> std::io::Result<()> { | |
| 9 | 55 | f.seek(pos).await?; |
| 10 | 56 | f.read_exact(buf).await |
| 11 | 57 | } |
| 12 | 58 | |
| 13 | -pub async fn get_sample( f :&mut File | |
| 59 | +async fn get_sample( f :&mut File | |
| 14 | 60 | , buf :&mut [u8]) -> std::io::Result<()> { |
| 15 | 61 | let file_len = f.metadata().await?.len(); |
| 16 | 62 | let chunk_size = buf.len() / 3; | ... | ... |
| ... | ... | @@ -23,10 +23,10 @@ impl Uuid { |
| 23 | 23 | } |
| 24 | 24 | } |
| 25 | 25 | |
| 26 | -impl TryFrom<Vec<u8>> for Uuid { | |
| 26 | +impl TryFrom<&[u8]> for Uuid { | |
| 27 | 27 | type Error = Error; |
| 28 | 28 | |
| 29 | - fn try_from(value: Vec<u8>) -> Result<Self, Self::Error> { | |
| 30 | - Ok(Self(uuid::Uuid::from_slice(value.as_slice())?)) | |
| 29 | + fn try_from(value: &[u8]) -> Result<Self, Self::Error> { | |
| 30 | + Ok(Self(uuid::Uuid::from_slice(value)?)) | |
| 31 | 31 | } |
| 32 | 32 | } | ... | ... |
Please
register
or
login
to post a comment