add robots.txt

robots.txt

@@ -0,0 +1,25 @@
+User-agent: *
+Disallow: /gogsadmin/*/commits/
+Disallow: /gogsadmin/*/src/
+Disallow: /.github/
+Disallow: /.phan/
+Disallow: /assets/
+Disallow: /backup/
+Disallow: /bin/
+Disallow: /cache/
+Disallow: /logs/
+Disallow: /system/
+Disallow: /tests/
+Disallow: /tmp/
+Disallow: /user/
+Disallow: /vendor/
+Disallow: /webserver-configs/
+Allow: /user/pages/
+Allow: /user/themes/
+Allow: /user/images/
+Allow: /
+Allow: /gogsadmin/
+Allow: /feed/
+Allow: *.css$
+Allow: *.js$
+Allow: /system/*.js$