Skip to content

Commit 344fda6

Browse files
crizCraigcrizCraig
authored
Merge pull request #3 from deepdrive/revert-2-revert-1-embedded-python
Revert "Revert "Bundle embedded Python interpreter under Linux and add CI build script""
2 parents ffcd1a8 + f493936 commit 344fda6

112 files changed

Lines changed: 106 additions & 17530 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Config/FilterPlugin.ini‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[FilterPlugin]
2+
; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and
3+
; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively.
4+
;
5+
; Examples:
6+
; /README.txt
7+
; /Extras/...
8+
; /Binaries/ThirdParty/*.dll
9+
/EmbeddedPython/...

‎EmbeddedPython/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

‎Packaging/package-linux.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
3+
docker run --rm -ti "-v$ROOTDIR:/hostdir" -w /hostdir adamrehn/ue4-full:4.21.1 python3 /hostdir/Packaging/package.py

‎Packaging/package.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env python3
2+
from ue4helpers import ConanUtils, FilesystemUtils, PlatformInfo, PluginPackager, SubprocessUtils, VersionHelpers
3+
from os.path import abspath, dirname, join
4+
5+
# Build a UE4-compatible version of CPython 3.6.8
6+
print('Building our embeddable Python distribution...')
7+
SubprocessUtils.run(['ue4', 'conan', 'update'])
8+
SubprocessUtils.run(['ue4', 'conan', 'build', 'python-ue4==3.6.8'])
9+
10+
# Bundle the custom-built Python distribution in our source tree
11+
print('Bundling our embeddable Python distribution...')
12+
root = dirname(dirname(abspath(__file__)))
13+
bundled = join(root, 'EmbeddedPython', PlatformInfo.identifier())
14+
ConanUtils.copy_package('python-ue4/3.6.8@adamrehn/4.21', bundled)
15+
16+
# Create our plugin packager
17+
packager = PluginPackager(
18+
root = root,
19+
version = VersionHelpers.from_git_commit(),
20+
archive = '{name}-{version}-{platform}'
21+
)
22+
23+
# Clean any previous build artifacts
24+
packager.clean()
25+
26+
# Package the plugin
27+
packager.package()
28+
29+
# Compress the packaged distribution
30+
archive = packager.archive()
31+
32+
# TODO: upload the archive to Amazon S3
33+
print('Created compressed archive "{}".'.format(archive))

‎Source/PythonAutomation/Public/PythonAutomationModule.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#pragma once
44

55
#include "CoreMinimal.h"
6-
#include "ModuleInterface.h"
6+
#include "Modules/ModuleManager.h"
77

88
class FPythonAutomationModule : public IModuleInterface
99
{

‎Source/UnrealEnginePython/Private/UnrealEnginePython.cpp‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,24 @@ const char *ue4_module_options = "linux_global_symbols";
5151
#include "Android/AndroidApplication.h"
5252
#endif
5353

54+
#include "Misc/Paths.h"
55+
#include "HAL/FileManager.h"
56+
57+
58+
//Deleter class for TUniquePtr<wchar_t> pointers that store the result of Py_DecodeLocale()
59+
class PyMemDeleter
60+
{
61+
public:
62+
PyMemDeleter() = default;
63+
PyMemDeleter(const PyMemDeleter&) = default;
64+
PyMemDeleter& operator=(const PyMemDeleter&) = default;
65+
~PyMemDeleter() = default;
66+
67+
void operator()(wchar_t* mem) const {
68+
PyMem_RawFree(mem);
69+
}
70+
};
71+
5472

5573
const char *UEPyUnicode_AsUTF8(PyObject *py_str)
5674
{
@@ -185,7 +203,7 @@ static void setup_stdout_stderr()
185203
" def isatty(self):\n"
186204
" return False\n"
187205
"sys.stdout = UnrealEngineOutput(unreal_engine.log)\n"
188-
"sys.stderr = UnrealEngineOutput(unreal_engine.log_error)\n"
206+
"sys.stderr = UnrealEngineOutput(unreal_engine.log)\n"
189207
"\n"
190208
"class event:\n"
191209
" def __init__(self, event_signature):\n"
@@ -473,7 +491,41 @@ void FUnrealEnginePythonModule::StartupModule()
473491
Py_SetPath(Py_DecodeLocale(TCHAR_TO_UTF8(*BasePythonPath), NULL));
474492
#endif
475493
#endif
476-
494+
495+
//Retrieve the paths to the Engine and project plugin directories
496+
FString enginePlugins = FPaths::ConvertRelativePathToFull(FPaths::EnginePluginsDir());
497+
FString projectPlugins = FPaths::ConvertRelativePathToFull(FPaths::ProjectPluginsDir());
498+
499+
//Attempt to detect the absolute path to our EmbeddedPython directory
500+
TArray<FString> matches;
501+
FString dirToFind = TEXT("EmbeddedPython");
502+
IFileManager& fileManager = IFileManager::Get();
503+
fileManager.FindFilesRecursive(matches, *enginePlugins, *dirToFind, false, true, false);
504+
fileManager.FindFilesRecursive(matches, *projectPlugins, *dirToFind, false, true, false);
505+
506+
//If we detected our embedded Python then set the name of the Python interpreter appropriately
507+
if (matches.Num() > 0)
508+
{
509+
//Make sure we use the correct version of the embedded interpreter for the host platform
510+
#if PLATFORM_LINUX
511+
#define _PLATFORM_NAME TEXT("Linux")
512+
#elif PLATFORM_MAC
513+
#define _PLATFORM_NAME TEXT("Mac")
514+
#elif PLATFORM_WINDOWS
515+
#define _PLATFORM_NAME TEXT("Windows")
516+
#else
517+
#define _PLATFORM_NAME TEXT("Unknown")
518+
#endif
519+
FString programName = FPaths::Combine(matches[0], _PLATFORM_NAME, TEXT("bin"), TEXT("python3"));
520+
UE_LOG(LogPython, Log, TEXT("Setting Python program name to %s"), *programName);
521+
#undef _PLATFORM_NAME
522+
523+
//Pass the interpreter path to Python, ensuring the memory containing the string outlives the interpreter
524+
static TUniquePtr<wchar_t, PyMemDeleter> programNamePtr;
525+
programNamePtr.Reset(Py_DecodeLocale(TCHAR_TO_UTF8(*programName), NULL));
526+
Py_SetProgramName(programNamePtr.Get());
527+
}
528+
477529
Py_Initialize();
478530

479531
#if PLATFORM_WINDOWS

‎Source/UnrealEnginePython/UnrealEnginePython.Build.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,8 @@ public UnrealEnginePython(TargetInfo Target)
258258
// PublicIncludePaths.Add("/usr/include");
259259
// PublicIncludePaths.Add("/home/a/src/deepdrive-sim-2.1/Plugins/UnrealEnginePython/linux/include");
260260
PublicAdditionalLibraries.Add(libsPath);
261-
261+
PublicAdditionalLibraries.AddRange(new string[] { "pthread", "dl", "util", "m" });
262+
RuntimeDependencies.Add(System.IO.Path.Combine(ModuleDirectory, "../../EmbeddedPython/..."), StagedFileType.NonUFS);
262263
}
263264
else
264265
{
@@ -335,7 +336,7 @@ private string DiscoverPythonPath(string[] knownPaths, string binaryPath)
335336
private string DiscoverLinuxPythonIncludesPath()
336337
{
337338
List<string> paths = new List<string>(linuxKnownIncludesPaths);
338-
string projPy35path = Path.Combine(ModuleDirectory, "../../linux", "python3.5m", "include");
339+
string projPy35path = Path.Combine(ModuleDirectory, "../../EmbeddedPython/Linux", "include", "python3.6m");
339340
System.Console.WriteLine("Adding project python include path: " + projPy35path);
340341
paths.Insert(0, projPy35path);
341342
paths.Insert(0, Path.Combine(ModuleDirectory, "../../Binaries", "Linux", "include"));
@@ -354,7 +355,7 @@ private string DiscoverLinuxPythonIncludesPath()
354355
private string DiscoverLinuxPythonLibsPath()
355356
{
356357
List<string> paths = new List<string>(linuxKnownLibsPaths);
357-
string projPy35path = Path.Combine(ModuleDirectory, "../../linux", "x86_64-linux-gnu", "libpython3.5m.so");
358+
string projPy35path = Path.Combine(ModuleDirectory, "../../EmbeddedPython/Linux", "lib", "libpython3.6m.a");
358359
System.Console.WriteLine("Adding project python lib path: " + projPy35path);
359360
paths.Insert(0, projPy35path);
360361
paths.Insert(0, Path.Combine(ModuleDirectory, "../../Binaries", "Linux", "lib"));

‎linux/python3.5m/include/ImDib.h‎

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎linux/python3.5m/include/ImPlatform.h‎

Lines changed: 0 additions & 80 deletions
This file was deleted.

0 commit comments

Comments
 (0)