What response data does RenderingTest want?

The RenderingTest object can be parented to TestService and will send data to localhost:8001 upon a TestService:Run() call. It sends a POST request to /api/v1/submit_test containing lots of rendering information, including a screenshot, CPU/GPU info, performance info, tickets, etc.

What response data is necessary to “let it know” that this test failed/succeeded? The Developer Hub article states that you should pass the strings "passed", "failed", or "golden":

However this is no longer accurate and passing these strings now causes the following to happen:

What response data should I return to tell Studio if the test passed or failed?

3 Likes

RenderingTests are not a publicly supported feature. We started building that system in 2018 to detect rendering bugs, and have been gradually improving the system and using it for more things internally. Some day, it may be ready for public use.

Unfortunately, there was internal miscommunication about this feature that resulted in usage instructions being posted on the dev hub. I’m sorry if you’ve built technology on this, but we don’t have the bandwidth for proper support at the moment. The protocol may continue to work in its current state for the near future, but as we continue to iterate on our internal system it may break for you.

We’ll update the dev hub with a note warning that this API should not be relied upon. If you have uses for this technology, please let us know what they are in a feature request so we can understand the demand and work on proper public support sooner.

6 Likes

It wasn’t so much a miscommunication as it was me just personally documenting it without oversight because I thought it was interesting. I still had a wiki volunteer mentality where I’d just document niche stuff.

The DevHub’s documentation has always been sort of a wild west, a mishmash of community contributed documentation and official documentation. It seems work is finally being done on this front though.

7 Likes

Bump… sorry I’m late for the party, but this is actually useful.

Turns out you do need to pass "passed", "failed", or "golden" through for a result of a RenderTest, but you cannot do this as a JSON response. It must be raw text with "Content-Type": "text/plain".

Here is an example of it working:

And a python3 server (definitely not written by AI because I could not be bothered) for your convienence:

#!/usr/bin/env python3

from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from datetime import datetime
import base64
import json
import re
import traceback

HOST = "127.0.0.1"
PORT = 8001

OUT_DIR = Path("renderingtest_dumps")
OUT_DIR.mkdir(exist_ok=True)

RESPONSE_TOKEN = "passed"

def now_id() -> str:
    return datetime.now().strftime("%Y%m%d_%H%M%S_%f")


def safe_name(value: str) -> str:
    value = str(value)
    value = re.sub(r"[^a-zA-Z0-9_.-]+", "_", value)
    return value[:120] or "unnamed"


def send_renderingtest_response(handler: BaseHTTPRequestHandler) -> None:
    body = RESPONSE_TOKEN.encode("utf-8")

    handler.send_response(200)
    handler.send_header("Content-Type", "text/plain")
    handler.send_header("Content-Length", str(len(body)))
    handler.send_header("Cache-Control", "no-cache")
    handler.send_header("Connection", "close")
    handler.end_headers()
    handler.wfile.write(body)


def send_text(handler: BaseHTTPRequestHandler, status: int, text: str) -> None:
    body = text.encode("utf-8")

    handler.send_response(status)
    handler.send_header("Content-Type", "text/plain; charset=utf-8")
    handler.send_header("Content-Length", str(len(body)))
    handler.send_header("Connection", "close")
    handler.end_headers()
    handler.wfile.write(body)


def maybe_decode_base64_image(value: str) -> bytes | None:
    if not isinstance(value, str):
        return None

    raw = value.strip()

    if raw.startswith("data:image/"):
        comma = raw.find(",")
        if comma != -1:
            raw = raw[comma + 1:]

    if len(raw) < 256:
        return None

    raw = re.sub(r"\s+", "", raw)

    try:
        decoded = base64.b64decode(raw, validate=False)
    except Exception:
        return None

    if decoded.startswith(b"\x89PNG\r\n\x1a\n"):
        return decoded

    if decoded.startswith(b"\xff\xd8\xff"):
        return decoded

    if decoded.startswith(b"RIFF") and b"WEBP" in decoded[:32]:
        return decoded

    return None


def walk_for_images(obj, out_dir: Path, prefix: str = "root") -> int:
    saved = 0

    if isinstance(obj, dict):
        for key, value in obj.items():
            saved += walk_for_images(value, out_dir, f"{prefix}_{safe_name(key)}")

    elif isinstance(obj, list):
        for index, value in enumerate(obj):
            saved += walk_for_images(value, out_dir, f"{prefix}_{index}")

    elif isinstance(obj, str):
        image_bytes = maybe_decode_base64_image(obj)

        if image_bytes is not None:
            if image_bytes.startswith(b"\x89PNG\r\n\x1a\n"):
                ext = "png"
            elif image_bytes.startswith(b"\xff\xd8\xff"):
                ext = "jpg"
            elif image_bytes.startswith(b"RIFF"):
                ext = "webp"
            else:
                ext = "bin"

            path = out_dir / f"{safe_name(prefix)}.{ext}"
            path.write_bytes(image_bytes)

            print(f"Saved image: {path}")
            saved += 1

    return saved


class RenderingTestHandler(BaseHTTPRequestHandler):
    # Use HTTP/1.0 to avoid weird keep-alive/body reuse behaviour.
    protocol_version = "HTTP/1.0"

    def log_message(self, fmt: str, *args) -> None:
        print(f"{self.address_string()} - {fmt % args}")

    def do_GET(self) -> None:
        if self.path in {"/", "/health", "/api/v1/health"}:
            send_text(self, 200, "RenderingTest receiver is alive\n")
            return

        send_text(self, 404, f"Unknown GET path: {self.path}\n")

    def do_POST(self) -> None:
        request_id = now_id()
        request_dir = OUT_DIR / request_id
        request_dir.mkdir(parents=True, exist_ok=True)

        try:
            content_length = int(self.headers.get("Content-Length", "0"))
            body = self.rfile.read(content_length)

            print()
            print("=== REQUEST ===")
            print("Path:", self.path)
            print("Content-Type:", self.headers.get("Content-Type"))
            print("Content-Length:", content_length)
            print("Body bytes:", len(body))

            try:
                print("Body preview:", body[:200].decode("utf-8", errors="replace"))
            except Exception:
                print("Body preview: <decode failed>")

            headers_path = request_dir / "headers.json"
            raw_path = request_dir / "raw_body.bin"

            headers_path.write_text(
                json.dumps(dict(self.headers), indent=4),
                encoding="utf-8",
            )

            raw_path.write_bytes(body)

            if self.path == "/api/v1/submit_test":
                self.handle_submit_test(request_dir, body)
                send_renderingtest_response(self)
                return

            # Funny enough, Roblox appears to have spelled this wrong.
            #   /api/v1/alltestfinised???
            if self.path == "/api/v1/alltestfinised":
                print("All rendering tests finished signal received.")
                print("Finish body:", body.decode("utf-8", errors="replace"))

                (request_dir / "finish_body.bin").write_bytes(body)

                send_renderingtest_response(self)
                return

            print("Unknown POST path:", self.path)

            # Still return a valid token so Roblox does not instantly nuke the run.
            send_renderingtest_response(self)

        except Exception:
            print("SERVER ERROR:")
            traceback.print_exc()

            try:
                send_renderingtest_response(self)
            except Exception:
                pass

    def handle_submit_test(self, request_dir: Path, body: bytes) -> None:
        content_type = self.headers.get("Content-Type", "")

        if "application/json" not in content_type:
            print("submit_test was not JSON; saved raw only.")
            return

        try:
            text = body.decode("utf-8")
            parsed = json.loads(text)
        except Exception as err:
            print("JSON parse failed:", err)
            return

        json_path = request_dir / "body.json"
        json_path.write_text(
            json.dumps(parsed, indent=4),
            encoding="utf-8",
        )

        print("Parsed JSON saved:", json_path)

        if isinstance(parsed, dict):
            summary = {}

            for key, value in parsed.items():
                if isinstance(value, (str, int, float, bool)) or value is None:
                    summary[key] = value
                elif isinstance(value, list):
                    summary[key] = f"<list len={len(value)}>"
                elif isinstance(value, dict):
                    summary[key] = f"<dict keys={len(value.keys())}>"
                else:
                    summary[key] = f"<{type(value).__name__}>"

            summary_path = request_dir / "summary.json"
            summary_path.write_text(
                json.dumps(summary, indent=4),
                encoding="utf-8",
            )

            print("Summary saved:", summary_path)

        images_saved = walk_for_images(parsed, request_dir, "renderingtest")

        if images_saved == 0:
            print("No base64 PNG/JPEG/WebP image fields found automatically.")
        else:
            print(f"Images saved: {images_saved}")


def main() -> None:
    server = ThreadingHTTPServer((HOST, PORT), RenderingTestHandler)

    print(f"Listening on http://{HOST}:{PORT}")
    print("Expected Roblox endpoints:")
    print("  POST /api/v1/submit_test")
    print("  POST /api/v1/alltestfinised")
    print()
    print("Response mode:")
    print(f"  HTTP 200 text/plain body: {RESPONSE_TOKEN!r}")
    print()
    print("Waiting for Roblox Studio RenderingTest POSTs...")

    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print()
        print("Stopping server.")
    finally:
        server.server_close()


if __name__ == "__main__":
    main()