[python] test infra improvements (#10219)

## Summary

Infrastructure overhaul for cleaner code organization and faster CI.

## New Folder Structure
```
http-client-python/
├── emitter/                        # TypeScript emitter (unchanged)
├── generator/pygen/                # Python generator (unchanged)
│
├── tests/                          # Consolidated test infrastructure
│   ├── tox.ini                     # Unified tox config with uv for faster installs
│   ├── conftest.py                 # Root pytest fixtures
│   ├── requirements/
│   │   ├── base.txt                # pytest, pytest-asyncio, coverage
│   │   ├── lint.txt                # pylint
│   │   ├── typecheck.txt           # mypy, pyright
│   │   ├── docs.txt                # sphinx, apiview
│   │   ├── azure.txt               # azure-core, azure-mgmt-core
│   │   └── unbranded.txt           # corehttp
│   ├── mock_api/                   # Mock API tests
│   │   ├── azure/                  # Azure-specific tests
│   │   ├── unbranded/              # Unbranded-specific tests
│   │   └── shared/                 # Shared test utilities
│   ├── unit/                       # Unit tests for pygen internals
│   └── generated/                  # Generated SDK packages
│       ├── azure/                  # Azure flavor packages
│       └── unbranded/              # Unbranded flavor packages
│
├── eng/scripts/                    # TypeScript-based tooling
│   ├── ci/
│   │   ├── run-tests.ts            # Parallel test orchestrator
│   │   ├── lint.ts                 # ESLint + pylint runner
│   │   ├── typecheck.ts            # mypy + pyright runner
│   │   ├── format.ts               # Prettier + Black runner
│   │   ├── regenerate.ts           # SDK regeneration
│   │   ├── regenerate-common.ts    # Shared regeneration utilities
│   │   ├── run_pylint.py           # Pylint helper for generated SDKs
│   │   ├── run_mypy.py             # Mypy helper for generated SDKs
│   │   ├── run_pyright.py          # Pyright helper for generated SDKs
│   │   └── config/                 # Tool configurations
│   │       ├── pylintrc
│   │       ├── mypy.ini
│   │       └── pyrightconfig.json
│   └── setup/
│       ├── build.ts                # Build pygen wheel
│       ├── install.ts              # Install dependencies
│       └── prepare.ts              # Prepare environment
│
└── dist/                           # Compiled emitter output
```                                                              
## Key Changes
### npm Scripts
| Command | Description |
|---------|-------------|
| `npm run test` | Run all tests (emitter + generator) |
| `npm run test:emitter` | TypeScript emitter tests (vitest) |
| `npm run test:generator` | Python generator tests (tox) |
| `npm run lint` | Lint emitter (ESLint) + pygen source (pylint) |
| `npm run lint:generated` | Lint generated SDK packages |
| `npm run typecheck` | Type check pygen source (mypy + pyright) |
| `npm run typecheck:generated` | Type check generated SDK packages |
| `npm run format` | Format emitter (Prettier) + pygen (Black) |
| `npm run format:generated` | Format generated SDK packages |
| `npm run regenerate` | Regenerate all SDK packages |
### Speed Improvements
- **uv package installer**: ~5-10x faster than pip for installing
generated packages
- **Parallel test execution**: Tests run in parallel across CPU cores
- **Separated source vs generated validation**: Default commands only
run on source code for faster dev loop

---------

Co-authored-by: iscai-msft <isabellavcai@gmail.com>

Author: iscai-msft

.chronus/changes/python-testInfraImprovements-2026-2-31-14-30-46.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/http-client-python"
+---
+
+Clean up test infrastructure, run in parallel, and use uv pip when acceptable. CI runs 2-3 x faster

.gitignore

@@ -228,8 +228,8 @@ BenchmarkDotnet.Artifacts/
 !packages/http-client-java/package-lock.json
 
 # python emitter
-packages/http-client-python/generator/test/**/generated/
-packages/http-client-python/generator/test/**/cadl-ranch-coverage.json
+packages/http-client-python/tests/**/generated/
+packages/http-client-python/tests/**/cadl-ranch-coverage.json
 !packages/http-client-python/package-lock.json
 packages/http-client-python/micropip.lock
 packages/http-client-python/venv_build_wheel/

cspell.yaml

@@ -276,6 +276,7 @@ words:
   - unassignable
   - Uncapitalize
   - uncollapsed
+  - unconfigure
   - undifferentiable
   - undoc
   - Ungroup
@@ -307,6 +308,7 @@ words:
   - WHATWG
   - WINDOWSARMVMIMAGE
   - WINDOWSVMIMAGE
+  - workerid
   - xiangyan
   - xiaofei
   - xlarge

packages/http-client-python/emitter/src/code-model.ts

@@ -25,14 +25,7 @@ import {
   emitPagingHttpMethod,
 } from "./http.js";
 import { PythonSdkContext } from "./lib.js";
-import {
-  KnownTypes,
-  disableGenerationMap,
-  emitEndpointType,
-  getType,
-  simpleTypesMap,
-  typesMap,
-} from "./types.js";
+import { KnownTypes, emitEndpointType, getType } from "./types.js";
 import { emitParamBase, getClientNamespace, getImplementation, getRootNamespace } from "./utils.js";
 
 function emitBasicMethod<TServiceOperation extends SdkServiceOperation>(
@@ -366,7 +359,7 @@ export function emitCodeModel(sdkContext: PythonSdkContext) {
       continue;
     }
     // filter out specific models not used in python, e.g., pageable models
-    if (disableGenerationMap.has(model)) {
+    if (sdkContext.__disableGenerationMap.has(model)) {
       continue;
     }
     // filter out core models
@@ -391,7 +384,7 @@ export function emitCodeModel(sdkContext: PythonSdkContext) {
   }
 
   // clear usage when a model is only used by paging
-  for (const type of typesMap.values()) {
+  for (const type of sdkContext.__typesMap.values()) {
     if (
       type["type"] === "model" &&
       type["referredByOperationType"] === ReferredByOperationTypes.PagingOnly &&
@@ -402,9 +395,9 @@ export function emitCodeModel(sdkContext: PythonSdkContext) {
   }
 
   codeModel["types"] = [
-    ...typesMap.values(),
+    ...sdkContext.__typesMap.values(),
     ...Object.values(KnownTypes),
-    ...simpleTypesMap.values(),
+    ...sdkContext.__simpleTypesMap.values(),
   ];
   codeModel["crossLanguagePackageId"] = ignoreDiagnostics(getCrossLanguagePackageId(sdkContext));
   if ((sdkContext.emitContext.options as any).flavor === "azure") {

packages/http-client-python/emitter/src/emitter.ts

@@ -10,7 +10,6 @@ import { emitCodeModel } from "./code-model.js";
 import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js";
 import { runPython3 } from "./run-python3.js";
-import { disableGenerationMap, simpleTypesMap, typesMap } from "./types.js";
 import { getRootNamespace, md2Rst } from "./utils.js";
 
 function addDefaultOptions(sdkContext: PythonSdkContext) {
@@ -59,6 +58,9 @@ async function createPythonSdkContext(
   return {
     ...sdkContext,
     __endpointPathParameters: [],
+    __typesMap: new Map(),
+    __simpleTypesMap: new Map(),
+    __disableGenerationMap: new Set(),
   };
 }
 
@@ -100,12 +102,6 @@ function walkThroughNodes(yamlMap: Record<string, any>): Record<string, any> {
   return yamlMap;
 }
 
-function cleanAllCache() {
-  typesMap.clear();
-  simpleTypesMap.clear();
-  disableGenerationMap.clear();
-}
-
 export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
   try {
     await onEmitMain(context);
@@ -124,9 +120,6 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
 }
 
 async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
-  // clean all cache to make sure emitter could work in watch mode
-  cleanAllCache();
-
   const program = context.program;
   const sdkContext = await createPythonSdkContext(context);
   const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..");
@@ -168,7 +161,7 @@ async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
       try {
         await runPython3(path.join(root, "/eng/scripts/setup/install.py"));
         await runPython3(path.join(root, "/eng/scripts/setup/prepare.py"));
-      } catch (error) {
+      } catch {
         // if the python env is not ready, we use pyodide instead
         resolvedOptions["use-pyodide"] = true;
       }
@@ -250,7 +243,7 @@ async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
       execSync(
         `${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`,
       );
-      checkForPylintIssues(outputDir, excludePattern);
+      await checkForPylintIssues(outputDir, excludePattern);
     }
   }
 }
@@ -269,7 +262,7 @@ async function setupPyodideCall(root: string) {
         if (lockAge > 300) {
           fs.unlinkSync(micropipLockPath);
         }
-      } catch (err) {
+      } catch {
         // ignore
       }
     }
@@ -288,14 +281,14 @@ async function setupPyodideCall(root: string) {
       fs.closeSync(fd);
       fs.unlinkSync(micropipLockPath);
       break;
-    } catch (err) {
+    } catch {
       await new Promise((resolve) => setTimeout(resolve, 1000));
     }
   }
   return pyodide;
 }
 
-function checkForPylintIssues(outputDir: string, excludePattern: string) {
+async function checkForPylintIssues(outputDir: string, excludePattern: string) {
   const excludeRegex = new RegExp(excludePattern);
 
   const shouldExcludePath = (filePath: string): boolean => {
@@ -304,9 +297,8 @@ function checkForPylintIssues(outputDir: string, excludePattern: string) {
     return excludeRegex.test(normalizedPath);
   };
 
-  const processFile = (filePath: string) => {
-    let fileContent;
-    fileContent = fs.readFileSync(filePath, "utf-8");
+  const processFile = async (filePath: string) => {
+    let fileContent = await fs.promises.readFile(filePath, "utf-8");
     const pylintDisables: string[] = [];
     const lineEnding = fileContent.includes("\r\n") && os.platform() === "win32" ? "\r\n" : "\n";
     const lines: string[] = fileContent.split(lineEnding);
@@ -321,32 +313,38 @@ function checkForPylintIssues(outputDir: string, excludePattern: string) {
         fileContent = lines[0].includes("pylint: disable=")
           ? [lines[0] + "," + pylintDisables.join(",")].concat(lines.slice(1)).join(lineEnding)
           : `# pylint: disable=${pylintDisables.join(",")}${lineEnding}` + fileContent;
+        await fs.promises.writeFile(filePath, fileContent);
       }
     }
-
-    fs.writeFileSync(filePath, fileContent);
   };
 
-  const walkDir = (dir: string) => {
+  const collectPythonFiles = async (dir: string): Promise<string[]> => {
     if (shouldExcludePath(dir)) {
-      return;
+      return [];
     }
 
-    const files = fs.readdirSync(dir);
-    files.forEach((file) => {
-      const filePath = path.join(dir, file);
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+
+    const promises = entries.map(async (entry) => {
+      const filePath = path.join(dir, entry.name);
 
       if (shouldExcludePath(filePath)) {
-        return;
+        return [];
       }
 
-      if (fs.statSync(filePath).isDirectory()) {
-        walkDir(filePath);
-      } else if (file.endsWith(".py")) {
-        processFile(filePath);
+      if (entry.isDirectory()) {
+        return collectPythonFiles(filePath);
+      } else if (entry.name.endsWith(".py")) {
+        return [filePath];
       }
+      return [];
     });
+
+    const results = await Promise.all(promises);
+    return results.flat();
   };
 
-  walkDir(outputDir);
+  // Collect all Python files first, then process in parallel
+  const pythonFiles = await collectPythonFiles(outputDir);
+  await Promise.all(pythonFiles.map(processFile));
 }

packages/http-client-python/emitter/src/lib.ts

@@ -1,5 +1,6 @@
 import {
   SdkContext,
+  SdkType,
   UnbrandedSdkEmitterOptions,
 } from "@azure-tools/typespec-client-generator-core";
 import { createTypeSpecLibrary, JSONSchemaType, paramMessage } from "@typespec/compiler";
@@ -28,6 +29,9 @@ export interface PythonEmitterOptions {
 
 export interface PythonSdkContext extends SdkContext<PythonEmitterOptions> {
   __endpointPathParameters: Record<string, any>[];
+  __typesMap: Map<SdkType, Record<string, any>>;
+  __simpleTypesMap: Map<string | null, Record<string, any>>;
+  __disableGenerationMap: Set<SdkType>;
 }
 
 export const PythonEmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> = {

packages/http-client-python/emitter/src/system-requirements.ts

@@ -16,8 +16,8 @@ const execute = (
       options.onCreate(cp);
     }
 
-    options.onStdOutData && cp.stdout.on("data", options.onStdOutData);
-    options.onStdErrData && cp.stderr.on("data", options.onStdErrData);
+    if (options.onStdOutData) cp.stdout.on("data", options.onStdOutData);
+    if (options.onStdErrData) cp.stderr.on("data", options.onStdErrData);
 
     let err = "";
     let out = "";
@@ -34,7 +34,7 @@ const execute = (
     cp.on("error", (err) => {
       reject(err);
     });
-    cp.on("close", (code, signal) =>
+    cp.on("close", (code, _signal) =>
       resolve({
         stdout: out,
         stderr: err,
@@ -110,7 +110,7 @@ const tryPython = async (
       `"${PRINT_PYTHON_VERSION_SCRIPT}"`,
     ]);
     return validateVersionRequirement(resolution, result.stdout.trim(), requirement);
-  } catch (e) {
+  } catch {
     return {
       error: true,
       ...resolution,