From e0b852dd6bc3ef50a013039cef01feacf045daec Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 7 Jun 2026 23:17:07 +0200 Subject: [PATCH] Initial commit - Datalab Lite source code --- Jenkinsfile | 57 ++++++++++ README.md | 18 +++ k8s.py | 204 +++++++++++++++++++++++++++++++++ main.py | 169 ++++++++++++++++++++++++++++ minio_client.py | 72 ++++++++++++ templates/index.html | 260 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 780 insertions(+) create mode 100644 Jenkinsfile create mode 100644 README.md create mode 100644 k8s.py create mode 100644 main.py create mode 100644 minio_client.py create mode 100644 templates/index.html diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..4c202d3 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,57 @@ +pipeline { + agent any + stages { + stage('Clone') { + steps { + echo '✅ Code cloné depuis Gitea' + } + } + stage('SonarQube Analysis') { + steps { + withSonarQubeEnv('sonarqube') { + sh ''' + sonar-scanner \ + -Dsonar.projectKey=datalab-lite \ + -Dsonar.projectName="Datalab Lite" \ + -Dsonar.sources=. \ + -Dsonar.inclusions="**/*.py" \ + -Dsonar.python.version=3.11 + ''' + } + } + } + stage('Quality Gate') { + steps { + waitForQualityGate abortPipeline: false + } + } + stage('Deploy') { + steps { + sh ''' + # Mettre à jour le ConfigMap code + kubectl create configmap datalab-lite-code \ + --from-file=main.py=main.py \ + --from-file=k8s.py=k8s.py \ + --from-file=minio_client.py=minio_client.py \ + -n datalab-lite \ + --dry-run=client -o yaml | kubectl apply -f - + + # Mettre à jour le ConfigMap templates + kubectl create configmap datalab-lite-templates \ + --from-file=index.html=templates/index.html \ + -n datalab-lite \ + --dry-run=client -o yaml | kubectl apply -f - + + # Redémarrer le pod + kubectl rollout restart deployment datalab-lite -n datalab-lite + + echo "✅ Datalab Lite déployé" + ''' + } + } + } + post { + success { echo '✅ Pipeline terminé avec succès !' } + failure { echo '❌ Pipeline échoué !' } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..44a71bd --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Datalab Lite + +Plateforme Data Science en libre-service déployée sur Kubernetes. + +## Services disponibles +- JupyterLab +- RStudio +- VSCode Server + +## Architecture +- Backend : FastAPI (Python 3.11) +- Auth : Keycloak OIDC +- Storage : MinIO S3 +- AI Chat : LiteLLM (llama3.2:3b) +- Deploy : Kubernetes + +## URL +https://datalablite.lab.leadwire.dev diff --git a/k8s.py b/k8s.py new file mode 100644 index 0000000..0635837 --- /dev/null +++ b/k8s.py @@ -0,0 +1,204 @@ +from kubernetes import client, config +import random, string +from minio_client import generate_password, MINIO_URL, MINIO_CONSOLE_URL +import hashlib + +def generate_service_password(username: str, service_type: str) -> str: + """Génère un password unique par user+service, persistant""" + seed = f"datalab-{username}-{service_type}-2026" + return hashlib.sha256(seed.encode()).hexdigest()[:12] + +S3_INIT_SCRIPT = ( + "pip install s3contents -q --target=/shared/packages && " + "mkdir -p /shared/jupyter && " + "printf '%s\\n' " + "'import sys' " + "'sys.path.insert(0, \"/shared/packages\")' " + "'import os' " + "'c.ServerApp.contents_manager_class = \"s3contents.S3ContentsManager\"' " + "'c.S3ContentsManager.endpoint_url = \"https://\" + os.environ.get(\"MINIO_ENDPOINT\", \"\")' " + "'c.S3ContentsManager.access_key_id = os.environ.get(\"MINIO_ACCESS_KEY\", \"\")' " + "'c.S3ContentsManager.secret_access_key = os.environ.get(\"MINIO_SECRET_KEY\", \"\")' " + "'c.S3ContentsManager.bucket = os.environ.get(\"MINIO_BUCKET\", \"\")' " + "> /shared/jupyter/jupyter_server_config.py && echo OK" +) + +S3_START_CMD = ( + "mkdir -p /home/jovyan/.jupyter && " + "cp /shared/jupyter/jupyter_server_config.py /home/jovyan/.jupyter/ && " + "export PYTHONPATH=/shared/packages:$PYTHONPATH && " + "start-notebook.sh" +) + + +try: + config.load_incluster_config() +except: + config.load_kube_config() + +DOMAIN = 'lab.leadwire.dev' + +def get_namespace(username): + return f"user-{username}" + +def random_id(length=5): + return ''.join(random.choices(string.digits, k=length)) + +def ensure_namespace(username): + v1 = client.CoreV1Api() + ns = get_namespace(username) + try: + v1.read_namespace(ns) + except: + v1.create_namespace(client.V1Namespace( + metadata=client.V1ObjectMeta( + name=ns, + labels={"managed-by": "datalab-lite", "user": username} + ) + )) + return ns + +def create_service(username, service_type, svc_config): + # Générer password unique par user+service + service_password = generate_service_password(username, service_type) + # Remplacer {token} dans les env vars + resolved_env = {} + for k, v in svc_config["env"].items(): + resolved_env[k] = v.replace("{token}", service_password) + svc_config = {**svc_config, "env": resolved_env} + ns = ensure_namespace(username) + uid = random_id() + name = f"{username}-{service_type}-{uid}" + port = svc_config["port"] + apps = client.AppsV1Api() + v1 = client.CoreV1Api() + net = client.NetworkingV1Api() + + minio_password = generate_password(username) + minio_envs = { + "MINIO_ENDPOINT": MINIO_URL, + "MINIO_ACCESS_KEY": username, + "MINIO_SECRET_KEY": minio_password, + "MINIO_BUCKET": f"user-{username}", + "AWS_ACCESS_KEY_ID": username, + "AWS_SECRET_ACCESS_KEY": minio_password, + "AWS_S3_ENDPOINT": f"https://{MINIO_URL}", + "NB_USER": username, + "PYTHONPATH": "/home/jovyan/jupyter-packages", + "PYTHONHTTPSVERIFY": "0", + } + env_vars = [client.V1EnvVar(name=k, value=v) for k, v in {**svc_config["env"], **minio_envs}.items()] + + deployment = client.V1Deployment( + metadata=client.V1ObjectMeta( + name=name, namespace=ns, + labels={"app": name, "user": username, "service-type": service_type, + "token": svc_config["env"].get("JUPYTER_TOKEN", svc_config["env"].get("PASSWORD", ""))} + ), + spec=client.V1DeploymentSpec( + replicas=1, + selector=client.V1LabelSelector(match_labels={"app": name}), + template=client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta(labels={"app": name}), + spec=client.V1PodSpec( + init_containers=([ + client.V1Container( + name="install-s3", + image=svc_config["image"], + command=["sh", "-c", S3_INIT_SCRIPT], + env=env_vars, + volume_mounts=[client.V1VolumeMount(name="shared", mount_path="/shared")] + ) + ] if service_type == "jupyter" else []), + containers=[ + client.V1Container( + name=name, + image=svc_config["image"], + ports=[client.V1ContainerPort(container_port=port)], + env=env_vars, + command=(["sh", "-c", S3_START_CMD] if service_type == "jupyter" else None), + volume_mounts=([client.V1VolumeMount(name="shared", mount_path="/shared")] if service_type == "jupyter" else []), + resources=client.V1ResourceRequirements( + requests={"memory": "512Mi", "cpu": "250m"}, + limits={"memory": "2Gi", "cpu": "1"} + ) + ) + ], + volumes=([client.V1Volume(name="shared", empty_dir=client.V1EmptyDirVolumeSource())] if service_type == "jupyter" else []) + ) + ) + ) + ) + apps.create_namespaced_deployment(ns, deployment) + svc = client.V1Service( + metadata=client.V1ObjectMeta(name=name, namespace=ns), + spec=client.V1ServiceSpec( + selector={"app": name}, + ports=[client.V1ServicePort(port=port, target_port=port)] + ) + ) + v1.create_namespaced_service(ns, svc) + host = f"{name}.{DOMAIN}" + ingress = client.V1Ingress( + metadata=client.V1ObjectMeta( + name=name, namespace=ns, + annotations={"cert-manager.io/cluster-issuer": "letsencrypt-prod", "nginx.ingress.kubernetes.io/proxy-read-timeout": "3600", "nginx.ingress.kubernetes.io/proxy-send-timeout": "3600"} + ), + spec=client.V1IngressSpec( + ingress_class_name="nginx", + rules=[client.V1IngressRule( + host=host, + http=client.V1HTTPIngressRuleValue(paths=[ + client.V1HTTPIngressPath( + path="/", path_type="Prefix", + backend=client.V1IngressBackend( + service=client.V1IngressServiceBackend( + name=name, + port=client.V1ServiceBackendPort(number=port) + ) + ) + ) + ]) + )], + tls=[client.V1IngressTLS(hosts=[host], secret_name=f"{name}-tls")] + ) + ) + net.create_namespaced_ingress(ns, ingress) + return f"https://{host}" + +def delete_service(username, service_name): + """service_name = nom complet ex: datauser-jupyter-12345""" + ns = get_namespace(username) + apps = client.AppsV1Api() + v1 = client.CoreV1Api() + net = client.NetworkingV1Api() + for fn in [ + lambda: apps.delete_namespaced_deployment(service_name, ns), + lambda: v1.delete_namespaced_service(service_name, ns), + lambda: net.delete_namespaced_ingress(service_name, ns), + ]: + try: fn() + except: pass + +def list_user_services(username): + ns = get_namespace(username) + apps = client.AppsV1Api() + result = [] + try: + deps = apps.list_namespaced_deployment( + ns, label_selector=f"user={username}" + ) + for d in deps.items: + stype = d.metadata.labels.get("service-type", "") + name = d.metadata.name + ready = d.status.ready_replicas or 0 + result.append({ + "name": name, + "type": stype, + "ready": ready > 0, + "url": f"https://{name}.{DOMAIN}", + "token": d.metadata.labels.get("token", "") + }) + except: + pass + return result diff --git a/main.py b/main.py new file mode 100644 index 0000000..bc5922f --- /dev/null +++ b/main.py @@ -0,0 +1,169 @@ +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse +from jinja2 import Environment, FileSystemLoader +import httpx, os, json +from k8s import create_service, delete_service, list_user_services +from minio_client import ensure_bucket + +app = FastAPI() +jinja_env = Environment(loader=FileSystemLoader("/app/code/templates")) + +KEYCLOAK_URL = "https://keycloak.lab.leadwire.dev/auth/realms/datalab" +CLIENT_ID = "datalab-lite" +CLIENT_SECRET= os.getenv("CLIENT_SECRET", "") +REDIRECT_URI = "https://datalablite.lab.leadwire.dev/callback" + +SERVICES = { + "jupyter": { + "name": "JupyterLab", + "image": "jupyter/scipy-notebook:latest", + "port": 8888, + "icon": "🔬", + "env": {"JUPYTER_ENABLE_LAB": "yes", "JUPYTER_TOKEN": "{token}"} + }, + "rstudio": { + "name": "RStudio", + "image": "rocker/rstudio:latest", + "port": 8787, + "icon": "📊", + "env": {"PASSWORD": "{token}", "USER": "rstudio"} + }, + "vscode": { + "name": "VSCode Server", + "image": "codercom/code-server:latest", + "port": 8080, + "icon": "💻", + "env": {"PASSWORD": "{token}"} + } +} + +def render(template_name, **ctx): + t = jinja_env.get_template(template_name) + return HTMLResponse(t.render(**ctx)) + +@app.get("/", response_class=HTMLResponse) +async def home(request: Request): + username = request.cookies.get("username") + if not username: + return RedirectResponse("/login") + services = list_user_services(username) + return render("index.html", username=username, services=services, catalog=SERVICES) + +@app.get("/login") +async def login(): + url = ( + f"{KEYCLOAK_URL}/protocol/openid-connect/auth" + f"?client_id={CLIENT_ID}" + f"&redirect_uri={REDIRECT_URI}" + f"&response_type=code" + f"&scope=openid profile email" + ) + return RedirectResponse(url) + +@app.get("/callback") +async def callback(code: str, request: Request): + async with httpx.AsyncClient(verify=False) as client: + resp = await client.post( + f"{KEYCLOAK_URL}/protocol/openid-connect/token", + data={ + "grant_type": "authorization_code", + "client_id": CLIENT_ID, + "client_secret": CLIENT_SECRET, + "code": code, + "redirect_uri": REDIRECT_URI, + } + ) + if resp.status_code != 200: + raise HTTPException(status_code=401, detail="Auth failed") + + tokens = resp.json() + access_token = tokens["access_token"] + + import base64 + payload = access_token.split(".")[1] + payload += "=" * (4 - len(payload) % 4) + user_info = json.loads(base64.b64decode(payload)) + username = user_info.get("preferred_username", "unknown") + + ensure_bucket(username) + + response = RedirectResponse("/", status_code=302) + response.set_cookie("token", access_token, httponly=True) + response.set_cookie("username", username, httponly=True) + return response + +@app.get("/logout") +async def logout(): + response = RedirectResponse("/login") + response.delete_cookie("token") + response.delete_cookie("username") + return response + +@app.post("/api/launch/{service_type}") +async def launch(service_type: str, request: Request): + username = request.cookies.get("username") + if not username: + raise HTTPException(status_code=401) + if service_type not in SERVICES: + raise HTTPException(status_code=400, detail="Service inconnu") + svc = SERVICES[service_type] + url = create_service(username, service_type, svc) + return JSONResponse({"status": "ok", "url": url}) + +@app.delete("/api/delete/{service_type}") +async def delete(service_type: str, request: Request): + username = request.cookies.get("username") + if not username: + raise HTTPException(status_code=401) + delete_service(username, service_type) + return JSONResponse({"status": "deleted"}) + +@app.get("/api/services") +async def services(request: Request): + username = request.cookies.get("username") + if not username: + raise HTTPException(status_code=401) + return JSONResponse(list_user_services(username)) + +@app.get("/api/minio-credentials") +async def minio_credentials(request: Request): + username = request.cookies.get("username") + if not username: + raise HTTPException(status_code=401) + from minio_client import get_minio_credentials + return JSONResponse(get_minio_credentials(username)) + +@app.get("/api/minio-credentials") +async def minio_credentials(request: Request): + username = request.cookies.get("username") + if not username: + raise HTTPException(status_code=401) + from minio_client import get_minio_credentials + return JSONResponse(get_minio_credentials(username)) + +@app.post("/api/chat") +async def chat(request: Request): + username = request.cookies.get("username") + if not username: + raise HTTPException(status_code=401) + body = await request.json() + messages = body.get("messages", []) + async with httpx.AsyncClient(verify=False, timeout=60) as client: + resp = await client.post( + "http://10.233.9.253:4000/v1/chat/completions", + headers={"Authorization": "Bearer sk-local123test", + "Content-Type": "application/json"}, + json={ + "model": "llama3.2:3b", + "messages": [ + {"role": "system", "content": f"Tu es un assistant expert Data Science pour {username}. Tu aides avec Python, pandas, SQL, machine learning, et l'utilisation de MinIO S3. Réponds en français de manière concise."} + ] + messages, + "temperature": 0.7, + "max_tokens": 512, + "stream": False + } + ) + if resp.status_code != 200: + raise HTTPException(status_code=502, detail="LiteLLM error") + result = resp.json() + return JSONResponse({"response": result["choices"][0]["message"]["content"]}) diff --git a/minio_client.py b/minio_client.py new file mode 100644 index 0000000..5bd254b --- /dev/null +++ b/minio_client.py @@ -0,0 +1,72 @@ +from minio import Minio +import os +import hashlib +import subprocess +import json + +MINIO_URL = "minio.lab.leadwire.dev" +MINIO_CONSOLE_URL = "https://minio-console.lab.leadwire.dev" +ADMIN_KEY = "admin" +ADMIN_SECRET = "Minio@Leadwire2026!" +MC_ALIAS = "minio-local" + +mc = Minio( + MINIO_URL, + access_key=ADMIN_KEY, + secret_key=ADMIN_SECRET, + secure=True, + cert_check=False +) + +def generate_password(username: str) -> str: + seed = f"datalab-{username}-2026" + return hashlib.sha256(seed.encode()).hexdigest()[:16] + +def mc_cmd(*args): + cmd = ["mc", "--insecure"] + list(args) + result = subprocess.run(cmd, capture_output=True, text=True) + print(f"mc {' '.join(args)}: code={result.returncode} out={result.stdout.strip()} err={result.stderr.strip()}") + return result.returncode == 0 + +def ensure_bucket(username): + bucket = f"user-{username}" + password = generate_password(username) + try: + # Créer le bucket + if not mc.bucket_exists(bucket): + mc.make_bucket(bucket) + print(f"Bucket {bucket} créé") + + # Créer le user MinIO + mc_cmd("admin", "user", "add", MC_ALIAS, username, password) + + # Créer la policy + policy = json.dumps({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": [ + f"arn:aws:s3:::{bucket}", + f"arn:aws:s3:::{bucket}/*" + ] + }] + }) + policy_file = f"/tmp/policy-{username}.json" + with open(policy_file, "w") as f: + f.write(policy) + + mc_cmd("admin", "policy", "create", MC_ALIAS, f"policy-{username}", policy_file) + mc_cmd("admin", "policy", "attach", MC_ALIAS, f"policy-{username}", "--user", username) + + print(f"User MinIO {username} configuré ✅") + except Exception as e: + print(f"MinIO error: {e}") + +def get_minio_credentials(username: str) -> dict: + return { + "url": MINIO_CONSOLE_URL, + "bucket": f"user-{username}", + "access_key": username, + "secret_key": generate_password(username) + } diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..8079cfc --- /dev/null +++ b/templates/index.html @@ -0,0 +1,260 @@ + + + + + +DTNum Labs + + + +
+
DTNum Labs
+ +
+ +
+ + +

Mes services actifs

+
+ {% if services %} + {% for svc in services %} +
+
{{ catalog[svc.type].icon if svc.type in catalog else '🔧' }}
+
{{ catalog[svc.type].name if svc.type in catalog else svc.type }}
+
+ + {% if svc.ready %}● En cours{% else %}⏳ Démarrage...{% endif %} + +
+ {% if svc.token %}
{{ svc.token }}
{% endif %}
+ {% if svc.ready %} + + {% endif %} + +
+
+ {% endfor %} + {% else %} +
Aucun service actif. Lance un service depuis le catalogue.
+ {% endif %} +
+ + +

Catalogue de services

+
+ {% for key, svc in catalog.items() %} +
+
{{ svc.icon }}
+
{{ svc.name }}
+
Port {{ svc.port }}
+ +
+ {% endfor %} +
+ + +
+ 🪣 Ton stockage S3 : s3://user-{{ username }}/ — + Voir les accès MinIO + +
+ +
+ + + + + + + + + + + + + +