diff --git a/README.md b/README.md index aa585e2d0..3a34cb672 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,10 @@ SELECT pgml.predict( # Installation PostgresML installation consists of three parts: PostgreSQL database, Postgres extension for machine learning and a dashboard app. The extension provides all the machine learning functionality and can be used independently using any SQL IDE. The dashboard app provides an easy to use interface for writing SQL notebooks, performing and tracking ML experiments and ML models. +## Serverless Cloud + +If you want to check out the functionality without the hassle of Docker, [sign up for a free PostgresML account](https://postgresml.org/signup). You'll get a free database in seconds, with access to GPUs and state of the art LLMs. + ## Docker ``` @@ -150,19 +154,14 @@ docker run \ sudo -u postgresml psql -d postgresml ``` -For more details, take a look at our [Quick Start with Docker](https://postgresml.org/docs/guides/setup/quick_start_with_docker) documentation. - -## Serverless Cloud - -If you want to check out the functionality without the hassle of Docker, [sign up for a free PostgresML account](https://postgresml.org/signup). You'll get a free database in seconds, with access to GPUs and state of the art LLMs. +For more details, take a look at our [Quick Start with Docker](https://postgresml.org/docs/guides/developer-docs/quick-start-with-docker) documentation. # Getting Started ## Option 1 -- On local installation, go to dashboard app at `http://localhost:8000/` to use SQL notebooks. - - On the cloud console click on the **Dashboard** button to connect to your instance with a SQL notebook, or connect directly with tools listed below. +- On local installation, go to dashboard app at `http://localhost:8000/` to use SQL notebooks. ## Option 2 diff --git a/docker/dashboard.sh b/docker/dashboard.sh index e4be965da..92b395674 100644 --- a/docker/dashboard.sh +++ b/docker/dashboard.sh @@ -4,6 +4,7 @@ set -e export DATABASE_URL=postgres://postgresml:postgresml@127.0.0.1:5432/postgresml export DASHBOARD_STATIC_DIRECTORY=/usr/share/pgml-dashboard/dashboard-static export DASHBOARD_CONTENT_DIRECTORY=/usr/share/pgml-dashboard/dashboard-content +export DASHBOARD_CONTENT_DOCS=/usr/share/pgml-docs export SEARCH_INDEX_DIRECTORY=/var/lib/pgml-dashboard/search-index export ROCKET_SECRET_KEY=$(openssl rand -hex 32) export ROCKET_ADDRESS=0.0.0.0 diff --git a/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service b/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service index 2e130814c..b2a1028a5 100644 --- a/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service +++ b/packages/postgresml-dashboard/etc/systemd/system/pgml-dashboard.service @@ -7,6 +7,7 @@ StartLimitIntervalSec=0 Environment=RUST_LOG=info Environment=DASHBOARD_STATIC_DIRECTORY=/usr/share/pgml-dashboard/dashboard-static Environment=DASHBOARD_CONTENT_DIRECTORY=/usr/share/pgml-dashboard/dashboard-content +Environment=DASHBOARD_CONTENT_DIRECTORY=/usr/share/pgml-docs Environment=ROCKET_ADDRESS=0.0.0.0 Environment=GITHUB_STARS=${GITHUB_STARS} Environment=SEARCH_INDEX_DIRECTORY=/var/lib/pgml-dashboard/search-index diff --git a/pgml-dashboard/.env.development b/pgml-dashboard/.env.development index 6129ccd80..81bf7e34a 100644 --- a/pgml-dashboard/.env.development +++ b/pgml-dashboard/.env.development @@ -1,2 +1,3 @@ DATABASE_URL=postgres:///pgml_dashboard_development DEV_MODE=true +RUST_LOG=debug,tantivy=error,rocket=info diff --git a/pgml-dashboard/Cargo.lock b/pgml-dashboard/Cargo.lock index ba9a3c5ef..0298a5519 100644 --- a/pgml-dashboard/Cargo.lock +++ b/pgml-dashboard/Cargo.lock @@ -559,6 +559,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.17.0" @@ -1741,6 +1750,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "markdown" +version = "1.0.0-alpha.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e9ce98969bb1391c8d6fdac320897ea7e86c4d356e8f220a5abd28b142e512" +dependencies = [ + "unicode-id", +] + [[package]] name = "markup5ever" version = "0.11.0" @@ -2186,6 +2204,7 @@ dependencies = [ "chrono", "comrak", "console-subscriber", + "convert_case", "csv-async", "dotenv", "env_logger", @@ -2193,6 +2212,7 @@ dependencies = [ "itertools", "lazy_static", "log", + "markdown", "num-traits", "once_cell", "parking_lot 0.12.1", @@ -2212,6 +2232,7 @@ dependencies = [ "tantivy", "time 0.3.23", "tokio", + "url", "yaml-rust", "zoomies", ] @@ -4083,6 +4104,12 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +[[package]] +name = "unicode-id" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1b6def86329695390197b82c1e244a54a131ceb66c996f2088a3876e2ae083f" + [[package]] name = "unicode-ident" version = "1.0.11" diff --git a/pgml-dashboard/Cargo.toml b/pgml-dashboard/Cargo.toml index 3313a16ff..172b49ddd 100644 --- a/pgml-dashboard/Cargo.toml +++ b/pgml-dashboard/Cargo.toml @@ -17,12 +17,14 @@ base64 = "0.21" comrak = "0.17" chrono = "0.4" csv-async = "1" +convert_case = "0.6" dotenv = "0.15" env_logger = "0.10" itertools = "0.10" parking_lot = "0.12" lazy_static = "1.4" log = "0.4" +markdown = "1.0.0-alpha.13" num-traits = "0.2" once_cell = "1.18" rand = "0.8" @@ -39,6 +41,7 @@ sqlx = { version = "0.6.3", features = [ "runtime-tokio-rustls", "postgres", "js tantivy = "0.19" time = "0.3" tokio = { version = "1", features = ["full"] } +url = "2.4" yaml-rust = "0.4" zoomies = { git="https://github.com/HyperparamAI/zoomies.git", branch="master" } pgvector = { version = "0.2.2", features = [ "sqlx", "postgres" ] } diff --git a/pgml-dashboard/README.md b/pgml-dashboard/README.md index a960ad77a..91cfdec00 100644 --- a/pgml-dashboard/README.md +++ b/pgml-dashboard/README.md @@ -2,4 +2,4 @@ PostgresML provides a dashboard with analytical views of the training data and model performance, as well as integrated notebooks for rapid iteration. It is primarily written in Rust using [Rocket](https://rocket.rs/) as a lightweight web framework and [SQLx](https://github.com/launchbadge/sqlx) to interact with the database. -Please see the [quick start instructions](https://postgresml.org/user_guides/setup/quick_start_with_docker/) for general information on installing or deploying PostgresML. A [developer guide](https://postgresml.org/developer_guide/overview/) is also available for those who would like to contribute. +Please see the [quick start instructions](https://postgresml.org/docs/guides/getting-started/sign-up) for general information on installing or deploying PostgresML. A [developer guide](https://postgresml.org/developer_guide/overview/) is also available for those who would like to contribute. diff --git a/pgml-dashboard/content/docs/guides/dashboard/overview.md b/pgml-dashboard/content/docs/guides/dashboard/overview.md index 4f0e16f43..70eb761f6 100644 --- a/pgml-dashboard/content/docs/guides/dashboard/overview.md +++ b/pgml-dashboard/content/docs/guides/dashboard/overview.md @@ -1,6 +1,6 @@ # Dashboard -PostgresML comes with a web app to provide visibility into models and datasets in your database. If you're running [our Docker container](/docs/guides/setup/quick_start_with_docker/), you can view it running on [http://localhost:8000/](http://localhost:8000/). +PostgresML comes with a web app to provide visibility into models and datasets in your database. If you're running [our Docker container](/docs/guides/developer-docs/quick-start-with-docker), you can view it running on [http://localhost:8000/](http://localhost:8000/). ## Generate example data diff --git a/pgml-dashboard/content/docs/guides/setup/distributed_training.md b/pgml-dashboard/content/docs/guides/setup/distributed_training.md index 41ff97e4f..748595f3c 100644 --- a/pgml-dashboard/content/docs/guides/setup/distributed_training.md +++ b/pgml-dashboard/content/docs/guides/setup/distributed_training.md @@ -22,7 +22,7 @@ psql \ -f dump.sql ``` -If you're using our Docker stack, you can import the data there:

+If you're using our Docker stack, you can import the data there:

``` psql \ diff --git a/pgml-dashboard/content/docs/guides/setup/v2/installation.md b/pgml-dashboard/content/docs/guides/setup/v2/installation.md index 683ad7302..3dd865f33 100644 --- a/pgml-dashboard/content/docs/guides/setup/v2/installation.md +++ b/pgml-dashboard/content/docs/guides/setup/v2/installation.md @@ -10,7 +10,7 @@ The extension can be installed by compiling it from source, or if you're using U !!! tip -If you're just looking to try PostgresML without installing it on your system, take a look at our [Quick Start with Docker](/docs/guides/setup/quick_start_with_docker) guide. +If you're just looking to try PostgresML without installing it on your system, take a look at our [Quick Start with Docker](/docs/guides/developer-docs/quick-start-with-docker) guide. !!! diff --git a/pgml-dashboard/src/api/docs.rs b/pgml-dashboard/src/api/docs.rs index bfabb78e3..10fc5d948 100644 --- a/pgml-dashboard/src/api/docs.rs +++ b/pgml-dashboard/src/api/docs.rs @@ -24,54 +24,36 @@ async fn search(query: &str, index: &State) -> ResponseOk ) } -#[get("/docs/", rank = 10)] -async fn doc_handler<'a>(path: PathBuf, cluster: &Cluster) -> Result { - let guides = vec![ - NavLink::new("Setup").children(vec![ - NavLink::new("Installation").children(vec![ - NavLink::new("v2").href("/docs/guides/setup/v2/installation"), - NavLink::new("Upgrade from v1.0 to v2.0") - .href("/docs/guides/setup/v2/upgrade-from-v1"), - NavLink::new("v1").href("/docs/guides/setup/installation"), - ]), - NavLink::new("Quick Start with Docker") - .href("/docs/guides/setup/quick_start_with_docker"), - NavLink::new("Distributed Training").href("/docs/guides/setup/distributed_training"), - NavLink::new("GPU Support").href("/docs/guides/setup/gpu_support"), - NavLink::new("Developer Setup").href("/docs/guides/setup/developers"), - ]), - NavLink::new("Training").children(vec![ - NavLink::new("Overview").href("/docs/guides/training/overview"), - NavLink::new("Algorithm Selection").href("/docs/guides/training/algorithm_selection"), - NavLink::new("Hyperparameter Search") - .href("/docs/guides/training/hyperparameter_search"), - NavLink::new("Preprocessing Data").href("/docs/guides/training/preprocessing"), - NavLink::new("Joint Optimization").href("/docs/guides/training/joint_optimization"), - ]), - NavLink::new("Predictions").children(vec![ - NavLink::new("Overview").href("/docs/guides/predictions/overview"), - NavLink::new("Deployments").href("/docs/guides/predictions/deployments"), - NavLink::new("Batch Predictions").href("/docs/guides/predictions/batch"), - ]), - NavLink::new("Transformers").children(vec![ - NavLink::new("Setup").href("/docs/guides/transformers/setup"), - NavLink::new("Pre-trained Models").href("/docs/guides/transformers/pre_trained_models"), - NavLink::new("Fine Tuning").href("/docs/guides/transformers/fine_tuning"), - NavLink::new("Embeddings").href("/docs/guides/transformers/embeddings"), - ]), - NavLink::new("Vector Operations").children(vec![ - NavLink::new("Overview").href("/docs/guides/vector_operations/overview") - ]), - NavLink::new("Dashboard").href("/docs/guides/dashboard/overview"), - NavLink::new("Schema").children(vec![ - NavLink::new("Models").href("/docs/guides/schema/models"), - NavLink::new("Snapshots").href("/docs/guides/schema/snapshots"), - NavLink::new("Projects").href("/docs/guides/schema/projects"), - NavLink::new("Deployments").href("/docs/guides/schema/deployments"), - ]), - ]; - - render(cluster, &path, guides, "Guides", &Path::new("docs")).await +use rocket::fs::NamedFile; + +#[get("/docs/guides/.gitbook/assets/", rank = 10)] +pub async fn gitbook_assets(path: PathBuf) -> Option { + let path = PathBuf::from(&config::docs_dir()) + .join("docs/guides/.gitbook/assets/") + .join(path); + + NamedFile::open(path).await.ok() +} + +#[get("/docs/", rank = 5)] +async fn doc_handler(path: PathBuf, cluster: &Cluster) -> Result { + let root = PathBuf::from("docs/guides/"); + let index_path = PathBuf::from(&config::docs_dir()) + .join(&root) + .join("SUMMARY.md"); + let contents = tokio::fs::read_to_string(&index_path).await.expect( + format!( + "could not read table of contents markdown: {:?}", + index_path + ) + .as_str(), + ); + let mdast = ::markdown::to_mdast(&contents, &::markdown::ParseOptions::default()) + .expect("could not parse table of contents markdown"); + let guides = markdown::parse_summary_into_nav_links(&mdast) + .expect("could not extract nav links from table of contents"); + + render(cluster, &path, guides, "Guides", &Path::new("docs"), &config::docs_dir()).await } #[get("/blog/", rank = 10)] @@ -134,6 +116,7 @@ async fn blog_handler<'a>(path: PathBuf, cluster: &Cluster) -> Result( mut nav_links: Vec, nav_title: &'a str, folder: &'a Path, + content: &'a str, ) -> Result { let url = path.clone(); // Get the document content - let path = Path::new(&config::content_dir()) + let path = Path::new(&content) .join(folder) .join(&(path.to_str().unwrap().to_string() + ".md")); + info!("path: {:?}", path); // Read to string let contents = match tokio::fs::read_to_string(&path).await { Ok(contents) => contents, @@ -244,7 +229,7 @@ async fn render<'a>( } pub fn routes() -> Vec { - routes![doc_handler, blog_handler, search] + routes![gitbook_assets, doc_handler, blog_handler, search] } #[cfg(test)] diff --git a/pgml-dashboard/src/components/navbar/template.html b/pgml-dashboard/src/components/navbar/template.html index 7f54386ae..f000aa67f 100644 --- a/pgml-dashboard/src/components/navbar/template.html +++ b/pgml-dashboard/src/components/navbar/template.html @@ -15,7 +15,7 @@