Added robots.txt (to be verified working)

src/pages/files.rs

@@ -17,3 +17,10 @@ async fn file(file: Path<(String, String)>, data: web::Data<AppState>) -> impl R
     };
     HttpResponse::Ok().body(bytes)
 }
+
+#[get("/robots.txt")]
+async fn robots() -> impl Responder {
+    "User-agent: *\
+    Disallow: /\
+    Allow: /$"
+}