LibWeb: Add start of HTML Tokenizer in Swift

Currently it's just a Token class.
Author: https://github.com/ADKaster Commit: fb074f9d0c Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/1130 Reviewed-by: https://github.com/AtkinsSJ Reviewed-by: https://github.com/alimpfard Reviewed-by: https://github.com/dzfrias
2025-07-28 11:49:44 +00:00 · 2024-08-17 20:16:16 -06:00 · 2024-08-17 20:16:16 -06:00 · fb074f9d0c · 2024-08-24 01:18:22 +00:00
commit fb074f9d0c
parent d0bc266c55
5 changed files with 169 additions and 1 deletions
--- a/Tests/LibWeb/TestHTMLTokenizerSwift.swift
+++ b/Tests/LibWeb/TestHTMLTokenizerSwift.swift
@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2024, Andrew Kaster <andrew@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+import AK
+import LibWeb
+import SwiftLibWeb
+import Foundation
+
+class StandardError: TextOutputStream {
+    func write(_ string: Swift.String) {
+        try! FileHandle.standardError.write(contentsOf: Data(string.utf8))
+    }
+}
+
+@main
+struct TestHTMLTokenizerSwift {
+
+    static func testTokenTypes() {
+        var standardError = StandardError()
+        print("Testing HTMLToken types...", to: &standardError)
+
+        let default_token = HTMLToken()
+        default_token.type = .Character(codePoint: "a")
+        precondition(default_token.isCharacter())
+
+        print("HTMLToken types pass", to: &standardError)
+    }
+
+    static func testParserWhitespace() {
+        var standardError = StandardError()
+        print("Testing HTMLToken parser whitespace...", to: &standardError)
+
+        for codePoint: Character in ["\t", "\n", "\r", "\u{000C}", " "] {
+            let token = HTMLToken(type: .Character(codePoint: codePoint))
+            precondition(token.isParserWhitespace())
+        }
+
+        for codePoint: Character in ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] {
+            let token = HTMLToken(type: .Character(codePoint: codePoint))
+            precondition(!token.isParserWhitespace())
+        }
+
+        print("HTMLToken parser whitespace pass", to: &standardError)
+    }
+
+    static func main() {
+        var standardError = StandardError()
+        print("Starting test suite...", to: &standardError)
+
+        testTokenTypes()
+        testParserWhitespace()
+
+        print("All tests pass", to: &standardError)
+    }
+}