Initial commit - Datalab Lite source code

Jenkinsfile

@@ -0,0 +1,57 @@
+pipeline {
+    agent any
+    stages {
+        stage('Clone') {
+            steps {
+                echo '✅ Code cloné depuis Gitea'
+            }
+        }
+        stage('SonarQube Analysis') {
+            steps {
+                withSonarQubeEnv('sonarqube') {
+                    sh '''
+                        sonar-scanner \
+                          -Dsonar.projectKey=datalab-lite \
+                          -Dsonar.projectName="Datalab Lite" \
+                          -Dsonar.sources=. \
+                          -Dsonar.inclusions="**/*.py" \
+                          -Dsonar.python.version=3.11
+                    '''
+                }
+            }
+        }
+        stage('Quality Gate') {
+            steps {
+                waitForQualityGate abortPipeline: false
+            }
+        }
+        stage('Deploy') {
+            steps {
+                sh '''
+                    # Mettre à jour le ConfigMap code
+                    kubectl create configmap datalab-lite-code \
+                        --from-file=main.py=main.py \
+                        --from-file=k8s.py=k8s.py \
+                        --from-file=minio_client.py=minio_client.py \
+                        -n datalab-lite \
+                        --dry-run=client -o yaml | kubectl apply -f -
+
+                    # Mettre à jour le ConfigMap templates
+                    kubectl create configmap datalab-lite-templates \
+                        --from-file=index.html=templates/index.html \
+                        -n datalab-lite \
+                        --dry-run=client -o yaml | kubectl apply -f -
+
+                    # Redémarrer le pod
+                    kubectl rollout restart deployment datalab-lite -n datalab-lite
+
+                    echo "✅ Datalab Lite déployé"
+                '''
+            }
+        }
+    }
+    post {
+        success { echo '✅ Pipeline terminé avec succès !' }
+        failure { echo '❌ Pipeline échoué !' }
+    }
+}

README.md

@@ -0,0 +1,18 @@
+# Datalab Lite
+
+Plateforme Data Science en libre-service déployée sur Kubernetes.
+
+## Services disponibles
+- JupyterLab
+- RStudio
+- VSCode Server
+
+## Architecture
+- Backend : FastAPI (Python 3.11)
+- Auth : Keycloak OIDC
+- Storage : MinIO S3
+- AI Chat : LiteLLM (llama3.2:3b)
+- Deploy : Kubernetes
+
+## URL
+https://datalablite.lab.leadwire.dev

k8s.py

@@ -0,0 +1,204 @@
+from kubernetes import client, config
+import random, string
+from minio_client import generate_password, MINIO_URL, MINIO_CONSOLE_URL
+import hashlib
+
+def generate_service_password(username: str, service_type: str) -> str:
+    """Génère un password unique par user+service, persistant"""
+    seed = f"datalab-{username}-{service_type}-2026"
+    return hashlib.sha256(seed.encode()).hexdigest()[:12]
+
+S3_INIT_SCRIPT = (
+    "pip install s3contents -q --target=/shared/packages && "
+    "mkdir -p /shared/jupyter && "
+    "printf '%s\\n' "
+    "'import sys' "
+    "'sys.path.insert(0, \"/shared/packages\")' "
+    "'import os' "
+    "'c.ServerApp.contents_manager_class = \"s3contents.S3ContentsManager\"' "
+    "'c.S3ContentsManager.endpoint_url = \"https://\" + os.environ.get(\"MINIO_ENDPOINT\", \"\")' "
+    "'c.S3ContentsManager.access_key_id = os.environ.get(\"MINIO_ACCESS_KEY\", \"\")' "
+    "'c.S3ContentsManager.secret_access_key = os.environ.get(\"MINIO_SECRET_KEY\", \"\")' "
+    "'c.S3ContentsManager.bucket = os.environ.get(\"MINIO_BUCKET\", \"\")' "
+    "> /shared/jupyter/jupyter_server_config.py && echo OK"
+)
+
+S3_START_CMD = (
+    "mkdir -p /home/jovyan/.jupyter && "
+    "cp /shared/jupyter/jupyter_server_config.py /home/jovyan/.jupyter/ && "
+    "export PYTHONPATH=/shared/packages:$PYTHONPATH && "
+    "start-notebook.sh"
+)
+
+
+try:
+    config.load_incluster_config()
+except:
+    config.load_kube_config()
+
+DOMAIN = 'lab.leadwire.dev'
+
+def get_namespace(username):
+    return f"user-{username}"
+
+def random_id(length=5):
+    return ''.join(random.choices(string.digits, k=length))
+
+def ensure_namespace(username):
+    v1 = client.CoreV1Api()
+    ns = get_namespace(username)
+    try:
+        v1.read_namespace(ns)
+    except:
+        v1.create_namespace(client.V1Namespace(
+            metadata=client.V1ObjectMeta(
+                name=ns,
+                labels={"managed-by": "datalab-lite", "user": username}
+            )
+        ))
+    return ns
+
+def create_service(username, service_type, svc_config):
+    # Générer password unique par user+service
+    service_password = generate_service_password(username, service_type)
+    # Remplacer {token} dans les env vars
+    resolved_env = {}
+    for k, v in svc_config["env"].items():
+        resolved_env[k] = v.replace("{token}", service_password)
+    svc_config = {**svc_config, "env": resolved_env}
+    ns   = ensure_namespace(username)
+    uid  = random_id()
+    name = f"{username}-{service_type}-{uid}"
+    port = svc_config["port"]
+    apps = client.AppsV1Api()
+    v1   = client.CoreV1Api()
+    net  = client.NetworkingV1Api()
+
+    minio_password = generate_password(username)
+    minio_envs = {
+        "MINIO_ENDPOINT": MINIO_URL,
+        "MINIO_ACCESS_KEY": username,
+        "MINIO_SECRET_KEY": minio_password,
+        "MINIO_BUCKET": f"user-{username}",
+        "AWS_ACCESS_KEY_ID": username,
+        "AWS_SECRET_ACCESS_KEY": minio_password,
+        "AWS_S3_ENDPOINT": f"https://{MINIO_URL}",
+        "NB_USER": username,
+        "PYTHONPATH": "/home/jovyan/jupyter-packages",
+        "PYTHONHTTPSVERIFY": "0",
+    }
+    env_vars = [client.V1EnvVar(name=k, value=v) for k, v in {**svc_config["env"], **minio_envs}.items()]
+
+    deployment = client.V1Deployment(
+        metadata=client.V1ObjectMeta(
+            name=name, namespace=ns,
+            labels={"app": name, "user": username, "service-type": service_type,
+                    "token": svc_config["env"].get("JUPYTER_TOKEN", svc_config["env"].get("PASSWORD", ""))}
+        ),
+        spec=client.V1DeploymentSpec(
+            replicas=1,
+            selector=client.V1LabelSelector(match_labels={"app": name}),
+            template=client.V1PodTemplateSpec(
+                metadata=client.V1ObjectMeta(labels={"app": name}),
+                spec=client.V1PodSpec(
+                    init_containers=([
+                        client.V1Container(
+                            name="install-s3",
+                            image=svc_config["image"],
+                            command=["sh", "-c", S3_INIT_SCRIPT],
+                            env=env_vars,
+                            volume_mounts=[client.V1VolumeMount(name="shared", mount_path="/shared")]
+                        )
+                    ] if service_type == "jupyter" else []),
+                    containers=[
+                        client.V1Container(
+                            name=name,
+                            image=svc_config["image"],
+                            ports=[client.V1ContainerPort(container_port=port)],
+                            env=env_vars,
+                            command=(["sh", "-c", S3_START_CMD] if service_type == "jupyter" else None),
+                            volume_mounts=([client.V1VolumeMount(name="shared", mount_path="/shared")] if service_type == "jupyter" else []),
+                            resources=client.V1ResourceRequirements(
+                                requests={"memory": "512Mi", "cpu": "250m"},
+                                limits={"memory": "2Gi", "cpu": "1"}
+                            )
+                        )
+                    ],
+                    volumes=([client.V1Volume(name="shared", empty_dir=client.V1EmptyDirVolumeSource())] if service_type == "jupyter" else [])
+                )
+            )
+        )
+    )
+    apps.create_namespaced_deployment(ns, deployment)
+    svc = client.V1Service(
+        metadata=client.V1ObjectMeta(name=name, namespace=ns),
+        spec=client.V1ServiceSpec(
+            selector={"app": name},
+            ports=[client.V1ServicePort(port=port, target_port=port)]
+        )
+    )
+    v1.create_namespaced_service(ns, svc)
+    host = f"{name}.{DOMAIN}"
+    ingress = client.V1Ingress(
+        metadata=client.V1ObjectMeta(
+            name=name, namespace=ns,
+            annotations={"cert-manager.io/cluster-issuer": "letsencrypt-prod", "nginx.ingress.kubernetes.io/proxy-read-timeout": "3600", "nginx.ingress.kubernetes.io/proxy-send-timeout": "3600"}
+        ),
+        spec=client.V1IngressSpec(
+            ingress_class_name="nginx",
+            rules=[client.V1IngressRule(
+                host=host,
+                http=client.V1HTTPIngressRuleValue(paths=[
+                    client.V1HTTPIngressPath(
+                        path="/", path_type="Prefix",
+                        backend=client.V1IngressBackend(
+                            service=client.V1IngressServiceBackend(
+                                name=name,
+                                port=client.V1ServiceBackendPort(number=port)
+                            )
+                        )
+                    )
+                ])
+            )],
+            tls=[client.V1IngressTLS(hosts=[host], secret_name=f"{name}-tls")]
+        )
+    )
+    net.create_namespaced_ingress(ns, ingress)
+    return f"https://{host}"
+
+def delete_service(username, service_name):
+    """service_name = nom complet ex: datauser-jupyter-12345"""
+    ns = get_namespace(username)
+    apps = client.AppsV1Api()
+    v1   = client.CoreV1Api()
+    net  = client.NetworkingV1Api()
+    for fn in [
+        lambda: apps.delete_namespaced_deployment(service_name, ns),
+        lambda: v1.delete_namespaced_service(service_name, ns),
+        lambda: net.delete_namespaced_ingress(service_name, ns),
+    ]:
+        try: fn()
+        except: pass
+
+def list_user_services(username):
+    ns = get_namespace(username)
+    apps = client.AppsV1Api()
+    result = []
+    try:
+        deps = apps.list_namespaced_deployment(
+            ns, label_selector=f"user={username}"
+        )
+        for d in deps.items:
+            stype = d.metadata.labels.get("service-type", "")
+            name  = d.metadata.name
+            ready = d.status.ready_replicas or 0
+            result.append({
+                "name":  name,
+                "type":  stype,
+                "ready": ready > 0,
+                "url":   f"https://{name}.{DOMAIN}",
+                    "token": d.metadata.labels.get("token", "")
+            })
+    except:
+        pass
+    return result

main.py

@@ -0,0 +1,169 @@
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
+from jinja2 import Environment, FileSystemLoader
+import httpx, os, json
+from k8s import create_service, delete_service, list_user_services
+from minio_client import ensure_bucket
+
+app = FastAPI()
+jinja_env = Environment(loader=FileSystemLoader("/app/code/templates"))
+
+KEYCLOAK_URL = "https://keycloak.lab.leadwire.dev/auth/realms/datalab"
+CLIENT_ID    = "datalab-lite"
+CLIENT_SECRET= os.getenv("CLIENT_SECRET", "")
+REDIRECT_URI = "https://datalablite.lab.leadwire.dev/callback"
+
+SERVICES = {
+    "jupyter": {
+        "name": "JupyterLab",
+        "image": "jupyter/scipy-notebook:latest",
+        "port": 8888,
+        "icon": "🔬",
+        "env": {"JUPYTER_ENABLE_LAB": "yes", "JUPYTER_TOKEN": "{token}"}
+    },
+    "rstudio": {
+        "name": "RStudio",
+        "image": "rocker/rstudio:latest",
+        "port": 8787,
+        "icon": "📊",
+        "env": {"PASSWORD": "{token}", "USER": "rstudio"}
+    },
+    "vscode": {
+        "name": "VSCode Server",
+        "image": "codercom/code-server:latest",
+        "port": 8080,
+        "icon": "💻",
+        "env": {"PASSWORD": "{token}"}
+    }
+}
+
+def render(template_name, **ctx):
+    t = jinja_env.get_template(template_name)
+    return HTMLResponse(t.render(**ctx))
+
+@app.get("/", response_class=HTMLResponse)
+async def home(request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        return RedirectResponse("/login")
+    services = list_user_services(username)
+    return render("index.html", username=username, services=services, catalog=SERVICES)
+
+@app.get("/login")
+async def login():
+    url = (
+        f"{KEYCLOAK_URL}/protocol/openid-connect/auth"
+        f"?client_id={CLIENT_ID}"
+        f"&redirect_uri={REDIRECT_URI}"
+        f"&response_type=code"
+        f"&scope=openid profile email"
+    )
+    return RedirectResponse(url)
+
+@app.get("/callback")
+async def callback(code: str, request: Request):
+    async with httpx.AsyncClient(verify=False) as client:
+        resp = await client.post(
+            f"{KEYCLOAK_URL}/protocol/openid-connect/token",
+            data={
+                "grant_type": "authorization_code",
+                "client_id": CLIENT_ID,
+                "client_secret": CLIENT_SECRET,
+                "code": code,
+                "redirect_uri": REDIRECT_URI,
+            }
+        )
+    if resp.status_code != 200:
+        raise HTTPException(status_code=401, detail="Auth failed")
+
+    tokens = resp.json()
+    access_token = tokens["access_token"]
+
+    import base64
+    payload = access_token.split(".")[1]
+    payload += "=" * (4 - len(payload) % 4)
+    user_info = json.loads(base64.b64decode(payload))
+    username = user_info.get("preferred_username", "unknown")
+
+    ensure_bucket(username)
+
+    response = RedirectResponse("/", status_code=302)
+    response.set_cookie("token", access_token, httponly=True)
+    response.set_cookie("username", username, httponly=True)
+    return response
+
+@app.get("/logout")
+async def logout():
+    response = RedirectResponse("/login")
+    response.delete_cookie("token")
+    response.delete_cookie("username")
+    return response
+
+@app.post("/api/launch/{service_type}")
+async def launch(service_type: str, request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        raise HTTPException(status_code=401)
+    if service_type not in SERVICES:
+        raise HTTPException(status_code=400, detail="Service inconnu")
+    svc = SERVICES[service_type]
+    url = create_service(username, service_type, svc)
+    return JSONResponse({"status": "ok", "url": url})
+
+@app.delete("/api/delete/{service_type}")
+async def delete(service_type: str, request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        raise HTTPException(status_code=401)
+    delete_service(username, service_type)
+    return JSONResponse({"status": "deleted"})
+
+@app.get("/api/services")
+async def services(request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        raise HTTPException(status_code=401)
+    return JSONResponse(list_user_services(username))
+
+@app.get("/api/minio-credentials")
+async def minio_credentials(request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        raise HTTPException(status_code=401)
+    from minio_client import get_minio_credentials
+    return JSONResponse(get_minio_credentials(username))
+
+@app.get("/api/minio-credentials")
+async def minio_credentials(request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        raise HTTPException(status_code=401)
+    from minio_client import get_minio_credentials
+    return JSONResponse(get_minio_credentials(username))
+
+@app.post("/api/chat")
+async def chat(request: Request):
+    username = request.cookies.get("username")
+    if not username:
+        raise HTTPException(status_code=401)
+    body = await request.json()
+    messages = body.get("messages", [])
+    async with httpx.AsyncClient(verify=False, timeout=60) as client:
+        resp = await client.post(
+            "http://10.233.9.253:4000/v1/chat/completions",
+            headers={"Authorization": "Bearer sk-local123test",
+                     "Content-Type": "application/json"},
+            json={
+                "model": "llama3.2:3b",
+                "messages": [
+                    {"role": "system", "content": f"Tu es un assistant expert Data Science pour {username}. Tu aides avec Python, pandas, SQL, machine learning, et l'utilisation de MinIO S3. Réponds en français de manière concise."}
+                ] + messages,
+                "temperature": 0.7,
+                "max_tokens": 512,
+                "stream": False
+            }
+        )
+    if resp.status_code != 200:
+        raise HTTPException(status_code=502, detail="LiteLLM error")
+    result = resp.json()
+    return JSONResponse({"response": result["choices"][0]["message"]["content"]})

minio_client.py

@@ -0,0 +1,72 @@
+from minio import Minio
+import os
+import hashlib
+import subprocess
+import json
+
+MINIO_URL = "minio.lab.leadwire.dev"
+MINIO_CONSOLE_URL = "https://minio-console.lab.leadwire.dev"
+ADMIN_KEY = "admin"
+ADMIN_SECRET = "Minio@Leadwire2026!"
+MC_ALIAS = "minio-local"
+
+mc = Minio(
+    MINIO_URL,
+    access_key=ADMIN_KEY,
+    secret_key=ADMIN_SECRET,
+    secure=True,
+    cert_check=False
+)
+
+def generate_password(username: str) -> str:
+    seed = f"datalab-{username}-2026"
+    return hashlib.sha256(seed.encode()).hexdigest()[:16]
+
+def mc_cmd(*args):
+    cmd = ["mc", "--insecure"] + list(args)
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    print(f"mc {' '.join(args)}: code={result.returncode} out={result.stdout.strip()} err={result.stderr.strip()}")
+    return result.returncode == 0
+
+def ensure_bucket(username):
+    bucket = f"user-{username}"
+    password = generate_password(username)
+    try:
+        # Créer le bucket
+        if not mc.bucket_exists(bucket):
+            mc.make_bucket(bucket)
+            print(f"Bucket {bucket} créé")
+
+        # Créer le user MinIO
+        mc_cmd("admin", "user", "add", MC_ALIAS, username, password)
+
+        # Créer la policy
+        policy = json.dumps({
+            "Version": "2012-10-17",
+            "Statement": [{
+                "Effect": "Allow",
+                "Action": ["s3:*"],
+                "Resource": [
+                    f"arn:aws:s3:::{bucket}",
+                    f"arn:aws:s3:::{bucket}/*"
+                ]
+            }]
+        })
+        policy_file = f"/tmp/policy-{username}.json"
+        with open(policy_file, "w") as f:
+            f.write(policy)
+
+        mc_cmd("admin", "policy", "create", MC_ALIAS, f"policy-{username}", policy_file)
+        mc_cmd("admin", "policy", "attach", MC_ALIAS, f"policy-{username}", "--user", username)
+
+        print(f"User MinIO {username} configuré ✅")
+    except Exception as e:
+        print(f"MinIO error: {e}")
+
+def get_minio_credentials(username: str) -> dict:
+    return {
+        "url": MINIO_CONSOLE_URL,
+        "bucket": f"user-{username}",
+        "access_key": username,
+        "secret_key": generate_password(username)
+    }