From 04288c3fda55a3f8a7838e1a313abf5b27913e72 Mon Sep 17 00:00:00 2001 From: Kbz-8 Date: Sat, 13 Dec 2025 16:39:41 +0100 Subject: [PATCH] new tst --- .github/workflows/Test.yml | 53 +- .github/workflows/TestNew.yml | 40 -- build.zig | 18 +- scripts/cts_logs_to_xml.py | 200 -------- scripts/cts_report_to_html.py | 887 ---------------------------------- scripts/log_parser.py | 218 --------- 6 files changed, 51 insertions(+), 1365 deletions(-) delete mode 100644 .github/workflows/TestNew.yml delete mode 100644 scripts/cts_logs_to_xml.py delete mode 100644 scripts/cts_report_to_html.py delete mode 100644 scripts/log_parser.py diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index b394480..939b4ad 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -21,18 +21,59 @@ jobs: - name: Zig Tests run: zig build test-soft - - name: Vulkan Conformance Test Suite - run: zig build test-conformance-soft + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + targets: x86_64-unknown-linux-gnu,wasm32-unknown-unknown - - name: Vulkan CTS report + - uses: Swatinem/rust-cache@v2 + with: + cache-all-crates: "true" + cache-on-failure: "false" + + - name: Install cargo tools + uses: cargo-bins/cargo-binstall@main + + - name: Install Dioxus + shell: bash run: | - pip install pandas - zig build test-conformance-soft-result-to-html + source $HOME/.cargo/env + cargo binstall dioxus-cli --no-confirm --force --version 0.7.2 + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Install deqp-runner + run: cargo install deqp-runner + + - name: Verify installations + shell: bash + run: | + source $HOME/.cargo/env + echo "Verifying tool installations..." + which dx && dx --version || echo "dioxus-cli not found" + which deqp-runner && deqp-runner --version || echo "deqp-runner not found" + + - name: Run Vulkan CTS + run: zig build cts-soft + continue-on-error: true + + - name: Verify tests + run: ls cts | grep "results.csv" + + - name: Cloning CTS viewer + run: git clone github.com/Kbz-8/VulkanCTSViewer/ + + - name: Build CTS viewer + env: + URL: https://vulkan-driver-cts-report.kbz8.me/ + run: | + cd VulkanCTSViewer + dx build --verbose --platform web --release - name: Publish to Cloudflare Pages uses: cloudflare/wrangler-action@v3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - command: pages deploy cts_report --project-name=vulkan-driver-cts-report + command: pages deploy VulkanCTSViewer/target/dx/vulkan-cts-analyzer/release/web/public --project-name=vulkan-driver-cts-report gitHubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/TestNew.yml b/.github/workflows/TestNew.yml deleted file mode 100644 index a4363fd..0000000 --- a/.github/workflows/TestNew.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Test (new) - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -jobs: - build_and_test: - runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, 'ci skip')" - - steps: - - uses: actions/checkout@v4 - - uses: mlugg/setup-zig@v2 - - - name: Building - run: zig build - - - name: Zig Tests - run: zig build test-soft - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - targets: x86_64-unknown-linux-gnu,wasm32-unknown-unknown - - - uses: Swatinem/rust-cache@v2 - with: - cache-all-crates: "true" - cache-on-failure: "false" - - - name: Install deqp-runner - run: cargo install deqp-runner - - - name: Run multithreaded master Vulkan CTS - run: zig build test-conformance-multithreaded-soft - continue-on-error: true diff --git a/build.zig b/build.zig index 01d93bc..f9ade51 100644 --- a/build.zig +++ b/build.zig @@ -218,19 +218,9 @@ fn addCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const Implemen run.addArg(b.fmt("--deqp-caselist-file={s}", .{mustpass})); } - const run_step = b.step(b.fmt("test-conformance-{s}{s}", .{ impl.name, if (gdb) "-gdb" else "" }), b.fmt("Run Vulkan conformance tests for libvulkan_{s}{s}", .{ impl.name, if (gdb) " within GDB" else "" })); + const run_step = b.step(b.fmt("raw-cts-{s}{s}", .{ impl.name, if (gdb) "-gdb" else "" }), b.fmt("Run Vulkan conformance tests for libvulkan_{s}{s}", .{ impl.name, if (gdb) " within GDB" else "" })); run_step.dependOn(&run.step); - if (!gdb) { - const run_to_xml = b.addSystemCommand(&[_][]const u8{ "python", "./scripts/cts_logs_to_xml.py", "./vk-cts-logs.qpa", "./vk-cts-logs.xml" }); - - const run_to_report = b.addSystemCommand(&[_][]const u8{ "python", "./scripts/cts_report_to_html.py", "./vk-cts-logs.xml", "vk-cts-report.html" }); - run_to_report.step.dependOn(&run_to_xml.step); - - const run_report_step = b.step(b.fmt("test-conformance-{s}-result-to-html", .{impl.name}), b.fmt("Run Vulkan conformance tests for libvulkan_{s} with a HTML report", .{impl.name})); - run_report_step.dependOn(&run_to_report.step); - } - return &run.step; } @@ -244,7 +234,7 @@ fn addMultithreadedCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *c }, })); - const mustpass_path = try cts.path("mustpass/master/vk-default.txt").getPath3(b, null).toString(b.allocator); + const mustpass_path = try cts.path("mustpass/1.0.0/vk-default.txt").getPath3(b, null).toString(b.allocator); const cts_exe_path = try cts_exe_name.getPath3(b, null).toString(b.allocator); const run = b.addSystemCommand(&[_][]const u8{"deqp-runner"}); @@ -252,7 +242,7 @@ fn addMultithreadedCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *c run.addArg("run"); run.addArg("--deqp"); - run.addArg(cts_exe_path); + run.addArg(b.fmt("{s}/{s}", .{ b.build_root.path.?, cts_exe_path })); run.addArg("--caselist"); run.addArg(mustpass_path); run.addArg("--output"); @@ -261,7 +251,7 @@ fn addMultithreadedCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *c run.addArg(b.fmt("--deqp-archive-dir={s}", .{try cts.path("").getPath3(b, null).toString(b.allocator)})); run.addArg(b.fmt("--deqp-vk-library-path={s}", .{b.getInstallPath(.lib, impl_lib.out_lib_filename)})); - const run_step = b.step(b.fmt("test-conformance-multithreaded-{s}", .{impl.name}), b.fmt("Run Vulkan conformance tests in a multithreaded environment for libvulkan_{s}", .{impl.name})); + const run_step = b.step(b.fmt("cts-{s}", .{impl.name}), b.fmt("Run Vulkan conformance tests in a multithreaded environment for libvulkan_{s}", .{impl.name})); run_step.dependOn(&run.step); return &run.step; diff --git a/scripts/cts_logs_to_xml.py b/scripts/cts_logs_to_xml.py deleted file mode 100644 index ca2f3c2..0000000 --- a/scripts/cts_logs_to_xml.py +++ /dev/null @@ -1,200 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# drawElements Quality Program utilities -# -------------------------------------- -# -# Copyright 2015 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import os -import sys -import codecs -import xml.dom.minidom -import xml.sax -import xml.sax.handler -from log_parser import BatchResultParser, StatusCode - -STYLESHEET_FILENAME = "testlog.xsl" -LOG_VERSION = '0.3.2' - -class BuildXMLLogHandler(xml.sax.handler.ContentHandler): - def __init__ (self, doc): - self.doc = doc - self.elementStack = [] - self.rootElements = [] - - def getRootElements (self): - return self.rootElements - - def pushElement (self, elem): - if len(self.elementStack) == 0: - self.rootElements.append(elem) - else: - self.getCurElement().appendChild(elem) - self.elementStack.append(elem) - - def popElement (self): - self.elementStack.pop() - - def getCurElement (self): - if len(self.elementStack) > 0: - return self.elementStack[-1] - else: - return None - - def startDocument (self): - pass - - def endDocument (self): - pass - - def startElement (self, name, attrs): - elem = self.doc.createElement(name) - for name in attrs.getNames(): - value = attrs.getValue(name) - elem.setAttribute(name, value) - self.pushElement(elem) - - def endElement (self, name): - self.popElement() - - def characters (self, content): - # Discard completely empty content - if len(content.strip()) == 0: - return - - # Append as text node (not pushed to stack) - if self.getCurElement() != None: - txt = self.doc.createTextNode(content) - self.getCurElement().appendChild(txt) - -class LogErrorHandler(xml.sax.handler.ErrorHandler): - def __init__ (self): - pass - - def error (self, err): - #print("error(%s)" % str(err)) - pass - - def fatalError (self, err): - #print("fatalError(%s)" % str(err)) - pass - - def warning (self, warn): - #print("warning(%s)" % str(warn)) - pass - -def findFirstElementByName (nodes, name): - for node in nodes: - if node.nodeName == name: - return node - chFound = findFirstElementByName(node.childNodes, name) - if chFound != None: - return chFound - return None - -# Normalizes potentially broken (due to crash for example) log data to XML element tree -def normalizeToXml (result, doc): - handler = BuildXMLLogHandler(doc) - errHandler = LogErrorHandler() - - xml.sax.parseString(result.log, handler, errHandler) - - rootNodes = handler.getRootElements() - - # Check if we have TestCaseResult - testCaseResult = findFirstElementByName(rootNodes, 'TestCaseResult') - if testCaseResult == None: - # Create TestCaseResult element - testCaseResult = doc.createElement('TestCaseResult') - testCaseResult.setAttribute('CasePath', result.name) - testCaseResult.setAttribute('CaseType', 'SelfValidate') # \todo [pyry] Not recoverable.. - testCaseResult.setAttribute('Version', LOG_VERSION) - rootNodes.append(testCaseResult) - - # Check if we have Result element - resultElem = findFirstElementByName(rootNodes, 'Result') - if resultElem == None: - # Create result element - resultElem = doc.createElement('Result') - resultElem.setAttribute('StatusCode', result.statusCode) - resultElem.appendChild(doc.createTextNode(result.statusDetails)) - testCaseResult.appendChild(resultElem) - - return rootNodes - -def logToXml (logFilePath, outFilePath): - # Initialize Xml Document - dstDoc = xml.dom.minidom.Document() - batchResultNode = dstDoc.createElement('BatchResult') - batchResultNode.setAttribute("FileName", os.path.basename(logFilePath)) - dstDoc.appendChild(batchResultNode) - - # Initialize dictionary for counting status codes - countByStatusCode = {} - for code in StatusCode.STATUS_CODES: - countByStatusCode[code] = 0 - - # Write custom headers - out = codecs.open(outFilePath, "wb", encoding="utf-8") - out.write("\n") - out.write("\n" % STYLESHEET_FILENAME) - - summaryElem = dstDoc.createElement('ResultTotals') - batchResultNode.appendChild(summaryElem) - - # Print the first line manually - out.write(dstDoc.toprettyxml().splitlines()[1]) - out.write("\n") - - parser = BatchResultParser() - parser.init(logFilePath) - logFile = open(logFilePath, 'rb') - - result = parser.getNextTestCaseResult(logFile) - while result is not None: - - countByStatusCode[result.statusCode] += 1 - rootNodes = normalizeToXml(result, dstDoc) - - for node in rootNodes: - - # Do not append TestResults to dstDoc to save memory. - # Instead print them directly to the file and add tabs manually. - for line in node.toprettyxml().splitlines(): - out.write("\t" + line + "\n") - - result = parser.getNextTestCaseResult(logFile) - - # Calculate the totals to add at the end of the Xml file - for code in StatusCode.STATUS_CODES: - summaryElem.setAttribute(code, "%d" % countByStatusCode[code]) - summaryElem.setAttribute('All', "%d" % sum(countByStatusCode.values())) - - # Print the test totals and finish the Xml Document" - for line in dstDoc.toprettyxml().splitlines()[2:]: - out.write(line + "\n") - - out.close() - logFile.close() - -if __name__ == "__main__": - if len(sys.argv) != 3: - print("%s: [test log] [dst file]" % sys.argv[0]) - sys.exit(-1) - - logToXml(sys.argv[1], sys.argv[2]) diff --git a/scripts/cts_report_to_html.py b/scripts/cts_report_to_html.py deleted file mode 100644 index db34501..0000000 --- a/scripts/cts_report_to_html.py +++ /dev/null @@ -1,887 +0,0 @@ -#!/usr/bin/env python3 - -""" -Credits to Arthur Vasseur for this script. - -https://github.com/ArthurVasseur/Vkd/blob/main/scripts/cts_report.py -""" - -import sys -import re -import os -import math -import shutil -import xml.etree.ElementTree as ET -import pandas as pd -from datetime import datetime -from collections import Counter - -PAGE_SIZE = 100 - -def parse_raw_log(log_text: str): - """Extract XML blocks from a raw CTS log file.""" - pattern = re.compile( - r']*>.*?', - re.DOTALL - ) - matches = pattern.findall(log_text) - return matches - - -def parse_xml_file(path: str): - """Extract nodes from a pure XML file.""" - tree = ET.parse(path) - root = tree.getroot() - return [ - ET.tostring(elem, encoding="unicode") - for elem in root.findall(".//TestCaseResult") - ] - - -def process_testcases(xml_blocks): - """Convert XML test blocks into structured rows.""" - rows = [] - - for block in xml_blocks: - elem = ET.fromstring(block) - - case = elem.attrib.get("CasePath", "unknown") - duration = elem.findtext("Number", default="0") - result = elem.find("Result").attrib.get("StatusCode", "UNKNOWN") - message = elem.findtext("Text", default="") - - rows.append({ - "Test Case": case, - "Duration (µs)": int(duration), - "Status": result, - "Message": message, - "RawMessage": message, - }) - - return rows - -def format_message_html(message: str) -> str: - """Format test message for HTML display with proper handling of newlines and tabs.""" - if not message: - return "" - - import html - import json - import textwrap - - try: - formatted = bytes(message, 'utf-8').decode('unicode_escape') - except Exception: - formatted = message - for old, new in [('\\n', '\n'), ('\\t', '\t'), ('\\r', '\r')]: - formatted = formatted.replace(old, new) - - formatted = textwrap.dedent(formatted).strip() - - try: - if formatted.strip().startswith('{') or formatted.strip().startswith('['): - parsed = json.loads(formatted) - formatted = json.dumps(parsed, indent=2) - escaped = html.escape(formatted) - return f'
View JSON
{escaped}
' - except (json.JSONDecodeError, ValueError): - pass - - if '\n' in formatted or '\t' in formatted or len(message) > 100: - escaped = html.escape(formatted) - return f'
View details
{escaped}
' - else: - return html.escape(formatted) - -def status_to_html(status: str) -> str: - cls = { - "Pass": "status-Pass", - "Fail": "status-Fail", - "NotSupported": "status-NotSupported", - }.get(status, "") - return f'{status}' - -def calculate_statistics(df): - """Calculate test statistics from the dataframe.""" - status_counts = Counter(df['Status']) - total_tests = len(df) - total_duration = df['Duration (µs)'].sum() - avg_duration = df['Duration (µs)'].mean() if total_tests > 0 else 0 - - pass_count = status_counts.get('Pass', 0) - fail_count = status_counts.get('Fail', 0) - not_supported_count = status_counts.get('NotSupported', 0) - other_count = total_tests - (pass_count + fail_count + not_supported_count) - - pass_rate = (pass_count / total_tests * 100) if total_tests > 0 else 0 - - return { - 'total': total_tests, - 'pass': pass_count, - 'fail': fail_count, - 'not_supported': not_supported_count, - 'other': other_count, - 'pass_rate': pass_rate, - 'total_duration_us': total_duration, - 'total_duration_ms': total_duration / 1000, - 'total_duration_s': total_duration / 1_000_000, - 'avg_duration_us': avg_duration, - } - -def generate_pie_chart_svg(stats): - """Generate a simple SVG pie chart for test results.""" - total = stats['total'] - if total == 0: - return "" - - pass_pct = stats['pass'] / total - fail_pct = stats['fail'] / total - not_supported_pct = stats['not_supported'] / total - other_pct = stats['other'] / total - - segments = [] - cumulative = 0 - - colors = { - 'pass': '#22c55e', - 'fail': '#f97373', - 'not_supported': '#eab308', - 'other': '#64748b' - } - - for name, pct, color in [ - ('Pass', pass_pct, colors['pass']), - ('Fail', fail_pct, colors['fail']), - ('Not Supported', not_supported_pct, colors['not_supported']), - ('Other', other_pct, colors['other']) - ]: - if pct > 0: - segments.append({ - 'name': name, - 'percentage': pct * 100, - 'start': cumulative, - 'end': cumulative + pct, - 'color': color - }) - cumulative += pct - - svg_paths = [] - radius = 80 - cx, cy = 100, 100 - - for seg in segments: - start_angle = seg['start'] * 2 * 3.14159 - end_angle = seg['end'] * 2 * 3.14159 - - x1 = cx + radius * cos_approx(start_angle) - y1 = cy + radius * sin_approx(start_angle) - x2 = cx + radius * cos_approx(end_angle) - y2 = cy + radius * sin_approx(end_angle) - - large_arc = 1 if (end_angle - start_angle) > 3.14159 else 0 - - path = f'M {cx} {cy} L {x1} {y1} A {radius} {radius} 0 {large_arc} 1 {x2} {y2} Z' - svg_paths.append(f'') - - return f''' - - {chr(10).join(svg_paths)} - - ''' - -def cos_approx(angle): - import math - return math.cos(angle) - -def sin_approx(angle): - import math - return math.sin(angle) - -def build_pagination(page_num: int, num_pages: int, base_output: str) -> str: - """ - base_output: basename used for files, e.g. 'report' -> report_page_1.html - """ - window = 2 # how many pages before/after the current one to show - links = [] - - # First / Prev - if page_num > 1: - links.append(f'First') - links.append( - f'Prev' - ) - else: - links.append('First') - links.append('Prev') - - # Page range around current - start_page = max(1, page_num - window) - end_page = min(num_pages, page_num + window) - - # Ellipsis before - if start_page > 1: - links.append('…') - - for p in range(start_page, end_page + 1): - if p == page_num: - links.append(f'{p}') - else: - links.append( - f'{p}' - ) - - # Ellipsis after - if end_page < num_pages: - links.append('…') - - # Next / Last - if page_num < num_pages: - links.append( - f'Next' - ) - links.append( - f'Last' - ) - else: - links.append('Next') - links.append('Last') - - return f""" - - """ - -def main(): - if len(sys.argv) != 3: - print("Usage: cts_report.py ") - sys.exit(1) - - input_path = sys.argv[1] - output_path = sys.argv[2] - - if not os.path.exists(input_path): - print(f"Error: input file not found: {input_path}") - sys.exit(1) - - # Detect input format - with open(input_path, "r", encoding="utf-8", errors="ignore") as f: - content = f.read() - - if " 1: - duration_str = f"{stats['total_duration_s']:.2f}s" - else: - duration_str = f"{stats['total_duration_ms']:.2f}ms" - - base_output = os.path.splitext(output_path)[0] # e.g. "report" - - for page_index in range(num_pages): - start = page_index * PAGE_SIZE - end = min(start + PAGE_SIZE, total_tests) - df_page = df.iloc[start:end] - - # recreate placeholders & table for this page - formatted_messages_page = formatted_messages[start:end] - df_page["Message"] = [f"__MSG_PLACEHOLDER_{i}__" for i in range(start, end)] - table_html = df_page.to_html( - index=False, - escape=False, - justify="center", - border=0, - classes="cts-table", - table_id="results-table" - ) - - # Replace placeholders for this chunk - for i in range(start, end): - table_html = table_html.replace( - f"__MSG_PLACEHOLDER_{i}__", formatted_messages[i] - ) - - # Page numbering (1-based for humans) - page_num = page_index + 1 - page_title_suffix = f" – Page {page_num}/{num_pages}" - - # Simple HTML navigation (pure HTML, no JS) - pagination_nav = build_pagination(page_num, num_pages, base_output) - - page_html = f""" - - - - -Vulkan CTS Report{page_title_suffix} - - - -
-
-
-
-

Vulkan CTS Report

-

Summary of test cases, status and timings

-
-
-
- - {generation_time} -
-
- - Total: {stats['total']} tests -
-
-
- -
-
-
-
- Passed -
-
{stats['pass']}
-
{stats['pass_rate']:.1f}% success rate
-
- -
-
-
- Failed -
-
{stats['fail']}
-
{(stats['fail'] / stats['total'] * 100) if stats['total'] > 0 else 0:.1f}% of total
-
- -
-
-
- Not Supported -
-
{stats['not_supported']}
-
{(stats['not_supported'] / stats['total'] * 100) if stats['total'] > 0 else 0:.1f}% of total
-
- -
-
-
- Duration -
-
{duration_str}
-
Avg: {stats['avg_duration_us']:.0f} µs/test
-
-
- -
- {pie_chart_svg} -
- - {pagination_nav} - -
- {table_html} -
- -
- Generated by - - cts_report.py - - at {generation_time} -
-
-
- - -""" - page_output_path = f"cts_report/{base_output}_page_{page_num}.html" - - os.makedirs(os.path.dirname(page_output_path), exist_ok=True) - with open(page_output_path, "w", encoding="utf-8") as f: - f.write(page_html) - - shutil.copy(f"cts_report/{base_output}_page_1.html", "cts_report/index.html") - - print(f"[OK] HTML report saved to: {output_path}") - print(f"\n--- Test Statistics ---") - print(f"Total tests: {stats['total']}") - print(f"Passed: {stats['pass']} ({stats['pass_rate']:.1f}%)") - print(f"Failed: {stats['fail']} ({(stats['fail'] / stats['total'] * 100) if stats['total'] > 0 else 0:.1f}%)") - print(f"Not Supported: {stats['not_supported']} ({(stats['not_supported'] / stats['total'] * 100) if stats['total'] > 0 else 0:.1f}%)") - if stats['other'] > 0: - print(f"Other: {stats['other']} ({(stats['other'] / stats['total'] * 100) if stats['total'] > 0 else 0:.1f}%)") - print(f"Total Duration: {duration_str}") - print(f"Average Duration: {stats['avg_duration_us']:.0f} µs/test") - - -if __name__ == "__main__": - main() diff --git a/scripts/log_parser.py b/scripts/log_parser.py deleted file mode 100644 index f35c072..0000000 --- a/scripts/log_parser.py +++ /dev/null @@ -1,218 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# drawElements Quality Program utilities -# -------------------------------------- -# -# Copyright 2015 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import shlex -import sys -import xml.dom.minidom - -class StatusCode: - PASS = 'Pass' - FAIL = 'Fail' - QUALITY_WARNING = 'QualityWarning' - COMPATIBILITY_WARNING = 'CompatibilityWarning' - PENDING = 'Pending' - NOT_SUPPORTED = 'NotSupported' - RESOURCE_ERROR = 'ResourceError' - INTERNAL_ERROR = 'InternalError' - CRASH = 'Crash' - TIMEOUT = 'Timeout' - - STATUS_CODES = [ - PASS, - FAIL, - QUALITY_WARNING, - COMPATIBILITY_WARNING, - PENDING, - NOT_SUPPORTED, - RESOURCE_ERROR, - INTERNAL_ERROR, - CRASH, - TIMEOUT - ] - STATUS_CODE_SET = set(STATUS_CODES) - - @staticmethod - def isValid (code): - return code in StatusCode.STATUS_CODE_SET - -class TestCaseResult: - def __init__ (self, name, statusCode, statusDetails, log): - self.name = name - self.statusCode = statusCode - self.statusDetails = statusDetails - self.log = log - - def __str__ (self): - return "%s: %s (%s)" % (self.name, self.statusCode, self.statusDetails) - -class ParseError(Exception): - def __init__ (self, filename, line, message): - self.filename = filename - self.line = line - self.message = message - - def __str__ (self): - return "%s:%d: %s" % (self.filename, self.line, self.message) - -def splitContainerLine (line): - if sys.version_info > (3, 0): - # In Python 3, shlex works better with unicode. - return shlex.split(line) - else: - # In Python 2, shlex works better with bytes, so encode and decode again upon return. - return [w.decode('utf-8') for w in shlex.split(line.encode('utf-8'))] - -def getNodeText (node): - rc = [] - for node in node.childNodes: - if node.nodeType == node.TEXT_NODE: - rc.append(node.data) - return ''.join(rc) - -class BatchResultParser: - def __init__ (self): - pass - - def parseFile (self, filename): - self.init(filename) - - f = open(filename, 'rb') - for line in f: - self.parseLine(line) - self.curLine += 1 - f.close() - - return self.testCaseResults - - def getNextTestCaseResult (self, file): - try: - del self.testCaseResults[:] - self.curResultText = None - - isNextResult = self.parseLine(next(file)) - while not isNextResult: - isNextResult = self.parseLine(next(file)) - - # Return the next TestCaseResult - return self.testCaseResults.pop() - - except StopIteration: - # If end of file was reached and there is no log left, the parsing finished successful (return None). - # Otherwise, if there is still log to be parsed, it means that there was a crash. - if self.curResultText: - return TestCaseResult(self.curCaseName, StatusCode.CRASH, StatusCode.CRASH, self.curResultText) - else: - return None - - def init (self, filename): - # Results - self.sessionInfo = [] - self.testCaseResults = [] - - # State - self.curResultText = None - self.curCaseName = None - - # Error context - self.curLine = 1 - self.filename = filename - - def parseLine (self, line): - # Some test shaders contain invalid characters. - text = line.decode('utf-8', 'ignore') - if len(text) > 0 and text[0] == '#': - return self.parseContainerLine(line) - elif self.curResultText != None: - self.curResultText += line - return None - # else: just ignored - - def parseContainerLine (self, line): - isTestCaseResult = False - # Some test shaders contain invalid characters. - text = line.decode('utf-8', 'ignore') - args = splitContainerLine(text) - if args[0] == "#sessionInfo": - if len(args) < 3: - print(args) - self.parseError("Invalid #sessionInfo") - self.sessionInfo.append((args[1], ' '.join(args[2:]))) - elif args[0] == "#beginSession" or args[0] == "#endSession": - pass # \todo [pyry] Validate - elif args[0] == "#beginTestCaseResult": - if len(args) != 2 or self.curCaseName != None: - self.parseError("Invalid #beginTestCaseResult") - self.curCaseName = args[1] - self.curResultText = b"" - elif args[0] == "#endTestCaseResult": - if len(args) != 1 or self.curCaseName == None: - self.parseError("Invalid #endTestCaseResult") - self.parseTestCaseResult(self.curCaseName, self.curResultText) - self.curCaseName = None - self.curResultText = None - isTestCaseResult = True - elif args[0] == "#terminateTestCaseResult": - if len(args) < 2 or self.curCaseName == None: - self.parseError("Invalid #terminateTestCaseResult") - statusCode = ' '.join(args[1:]) - statusDetails = statusCode - - if not StatusCode.isValid(statusCode): - # Legacy format - if statusCode == "Watchdog timeout occurred.": - statusCode = StatusCode.TIMEOUT - else: - statusCode = StatusCode.CRASH - - # Do not try to parse at all since XML is likely broken - self.testCaseResults.append(TestCaseResult(self.curCaseName, statusCode, statusDetails, self.curResultText)) - - self.curCaseName = None - self.curResultText = None - isTestCaseResult = True - else: - # Assume this is result text - if self.curResultText != None: - self.curResultText += line - - return isTestCaseResult - - def parseTestCaseResult (self, name, log): - try: - # The XML parser has troubles with invalid characters deliberately included in the shaders. - # This line removes such characters before calling the parser - log = log.decode('utf-8','ignore').encode("utf-8") - doc = xml.dom.minidom.parseString(log) - resultItems = doc.getElementsByTagName('Result') - if len(resultItems) != 1: - self.parseError("Expected 1 , found %d" % len(resultItems)) - - statusCode = resultItems[0].getAttributeNode('StatusCode').nodeValue - statusDetails = getNodeText(resultItems[0]) - except Exception as e: - statusCode = StatusCode.INTERNAL_ERROR - statusDetails = "XML parsing failed: %s" % str(e) - - self.testCaseResults.append(TestCaseResult(name, statusCode, statusDetails, log)) - - def parseError (self, message): - raise ParseError(self.filename, self.curLine, message)