Repository navigation
Expand file tree
/
Copy pathfinder.py
More file actions
289 lines (246 loc) · 12.8 KB
/
Copy pathfinder.py
File metadata and controls
289 lines (246 loc) · 12.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
"""
External dependency finding functionality.
This module provides the ExternalDependencyFinder class which identifies
files and directories that are imported from outside the source directory
and need to be temporarily copied during the build process.
"""
from __future__ import annotations
from pathlib import Path
from .analyzer import ImportAnalyzer
from .types import ExternalDependency
class ExternalDependencyFinder:
"""
Finds external dependencies that need to be copied.
This class analyzes Python files to identify imports that reference
modules outside the source directory. It determines which files or
directories need to be copied and where they should be placed.
Attributes:
project_root: Root directory of the project
src_dir: Source directory where the package code lives
analyzer: ImportAnalyzer instance for analyzing imports
"""
def __init__(
self,
project_root: Path,
src_dir: Path,
exclude_patterns: list[str] | None = None,
original_src_dir: Path | None = None,
) -> None:
"""
Initialize the dependency finder.
Args:
project_root: Root directory of the project
src_dir: Source directory to analyze (may be temp directory for subfolder builds)
exclude_patterns: Additional patterns to exclude (default: common sandbox patterns)
original_src_dir: Original source directory before any changes (e.g., before temp directory creation).
Used for relative path checks. If not provided, uses src_dir.
"""
self.project_root = project_root.resolve()
self.src_dir = src_dir.resolve()
# Store original src_dir for relative path checks (important for subfolder builds)
self.original_src_dir = (original_src_dir or src_dir).resolve()
self.analyzer = ImportAnalyzer(project_root)
# Patterns for directories/files to exclude (sandbox, skip, etc.)
default_patterns = [
"_SS",
"__SS",
"_sandbox",
"__sandbox",
"_skip",
"__skip",
"_test",
"__test__",
]
self.exclude_patterns = default_patterns + (exclude_patterns or [])
def find_external_dependencies(self, python_files: list[Path]) -> list[ExternalDependency]:
"""
Find all external dependencies that need to be copied.
Analyzes all provided Python files, classifies their imports,
and identifies which external files/directories need to be copied
into the source directory.
Args:
python_files: List of Python file paths to analyze
Returns:
List of ExternalDependency objects representing files/directories
that need to be copied
"""
external_deps: list[ExternalDependency] = []
seen_paths: set[Path] = set()
for file_path in python_files:
imports = self.analyzer.extract_imports(file_path)
for imp in imports:
self.analyzer.classify_import(imp, self.src_dir)
if imp.classification == "external" and imp.resolved_path:
source_path = imp.resolved_path
# Skip excluded paths (sandbox directories, etc.)
if self._should_exclude_path(source_path):
continue
# For files, only copy parent directory if it's a package
# Otherwise, copy just the individual file
if source_path.is_file():
parent_dir = source_path.parent
# Never copy the src/ directory itself - only copy individual files at its root
# This prevents copying the entire src/ directory (with all subdirectories)
# when only a file like _globals.py is needed
if parent_dir == self.project_root / "src":
# For files at root of src/, only copy the file, not the directory
track_path = source_path
else:
# Only copy parent directory if:
# 1. It's a package (has __init__.py), OR
# 2. Files from it are actually imported (which is the case here)
# But only copy the immediate parent, not entire directory trees
parent_is_package = (parent_dir / "__init__.py").exists()
files_are_imported = True # Always true when processing an import
# Only copy immediate parent directory, not grandparent directories
# This prevents copying entire trees like models/Information_extraction
# when we only need models/Information_extraction/_shared_ie
# Use original_src_dir for relative path checks to correctly handle
# subfolder builds where src_dir may point to temp directory
should_copy_dir = (
not self._should_exclude_path(parent_dir)
and (
parent_is_package or files_are_imported
) # Package OR files imported
and not parent_dir.is_relative_to(self.src_dir)
and not self.original_src_dir.is_relative_to(parent_dir)
and parent_dir != self.project_root
and parent_dir != self.project_root.parent
)
if should_copy_dir:
# Copy the directory instead of just the file
track_path = parent_dir
source_path = parent_dir
else:
# Copy just the file
track_path = source_path
elif source_path.is_dir():
# Don't copy directories that contain src_dir
if self.src_dir.is_relative_to(source_path):
continue
track_path = source_path
else:
continue
if track_path in seen_paths:
continue
seen_paths.add(track_path)
# Determine target path within src_dir
target_path = self._determine_target_path(source_path, imp.module_name)
if target_path:
# Only add if source is actually outside src_dir
if not source_path.is_relative_to(self.src_dir):
external_deps.append(
ExternalDependency(
source_path=source_path,
target_path=target_path,
import_name=imp.module_name,
file_path=file_path,
)
)
return external_deps
def _determine_target_path(self, source_path: Path, module_name: str) -> Path | None:
"""
Determine where an external file should be copied within src_dir.
Preserves the original directory structure relative to project_root or src
to maintain import paths like `from models. ...` or `from data. ...`.
Args:
source_path: Path to the source file or directory
module_name: Module name from the import statement
Returns:
Target path within src_dir, or None if cannot be determined
"""
if not source_path.exists():
return None
module_parts = module_name.split(".")
# Determine the base directory for calculating relative paths
# If source is under src/, use src/ as base to preserve structure
src_base = self.project_root / "src"
if src_base.exists() and source_path.is_relative_to(src_base):
# Source is under src/, preserve the path structure relative to src/
# This ensures imports like `from models. ...` work correctly
try:
relative_path = source_path.relative_to(src_base)
target = self.src_dir / relative_path
return target
except ValueError:
pass
# For sources not under src/, try to preserve structure based on module name
# Check if the module name structure matches the source path structure
if len(module_parts) > 1:
# For directories, check if the module path (excluding the last part for files) matches
if source_path.is_dir():
# For directories, check if module parts match the directory structure
# e.g., module "folder_structure.utility_folder.some_utility" with dir "folder_structure/utility_folder"
# should preserve "folder_structure/utility_folder" structure
# Calculate relative path from project_root
try:
relative_path = source_path.relative_to(self.project_root)
# If the module starts with the same parts as the relative path, preserve structure
relative_parts = list(relative_path.parts)
# Check if module_parts[:-1] (excluding the file/module name) matches relative_parts
if len(module_parts) > len(relative_parts):
# Module has more parts (includes the file name), check if the directory parts match
module_dir_parts = module_parts[: len(relative_parts)]
if module_dir_parts == relative_parts:
# Preserve the full structure
target = self.src_dir / relative_path
return target
# Try matching from the end
if len(relative_parts) <= len(module_parts):
# Check if the last parts of module match the relative path
for i in range(1, min(len(relative_parts), len(module_parts)) + 1):
if module_parts[-i:] == relative_parts[-i:]:
# Match found, preserve structure based on module name
target = self.src_dir / "/".join(
module_parts[: -i + 1 if i > 1 else len(module_parts)]
)
return target
except ValueError:
pass
# Fallback: preserve structure based on module name (excluding last part for files)
target = self.src_dir / "/".join(module_parts[:-1])
return target
else:
# For files, preserve structure based on module name (excluding filename)
target = self.src_dir / "/".join(module_parts[:-1]) / source_path.name
return target
# Simple top-level import - copy directly to src_dir
# This handles cases like `from some_globals import ...`
if source_path.is_file():
target = self.src_dir / source_path.name
else:
target = self.src_dir / source_path.name
return target
def _should_exclude_path(self, path: Path) -> bool:
"""
Check if a path should be excluded from copying.
Excludes paths that match common sandbox/skip patterns like _SS, __SS, etc.
Args:
path: Path to check
Returns:
True if the path should be excluded, False otherwise
"""
# Check each component of the path
for part in path.parts:
for pattern in self.exclude_patterns:
# Match if part equals pattern or starts with pattern
if part == pattern or part.startswith(pattern):
return True
return False
def _find_main_package(self) -> Path | None:
"""
Find the main package directory within src_dir.
Looks for directories containing __init__.py files, which indicate
Python packages.
Returns:
Path to the main package directory, or None if not found
"""
if not self.src_dir.exists():
return None
# Look for directories with __init__.py
package_dirs = [
d for d in self.src_dir.iterdir() if d.is_dir() and (d / "__init__.py").exists()
]
if package_dirs:
return package_dirs[0]
return None