Skip to content

Commit 5dc40f6

Browse files
ProgramComputerProgramComputer
committed
0.1.0-alpha3
1 parent 0b7552c commit 5dc40f6

47 files changed

Lines changed: 2682 additions & 2347 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ poetry.toml
243243

244244
# ruff
245245
.ruff_cache/
246-
vendor/
246+
vendor
247247

248248
# LSP config files
249249
pyrightconfig.json

‎.gitmodules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,6 @@
3838
[submodule "vendor/aws/aws-c-sdkutils"]
3939
path = vendor/aws/aws-c-sdkutils
4040
url = https://github.com/awslabs/aws-c-sdkutils.git
41+
[submodule "cwalk"]
42+
path = cwalk
43+
url = https://github.com/likle/cwalk

‎CONTRIBUTING.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,42 @@ We love your input! We want to make contributing to Embedding Bridge as easy and
2626
2. Update the documentation if you're introducing new features.
2727
3. The PR will be merged once you have the sign-off of at least one maintainer.
2828

29+
## Helper Scripts
30+
31+
The `scripts/` directory contains helper scripts to automate building and installing native dependencies:
32+
33+
- **build_aws.sh**: clones and builds all AWS C SDK libraries into `vendor/aws/install`.
34+
- **build_arrow.sh**: downloads and builds Apache Arrow C++ (and optional GLib bindings via `--with-glib`), installing to `/usr/local` or the `ARROW_INSTALL_DIR` defined in this repository.
35+
- **build_aws.md**: a human-readable guide describing the manual steps performed by `build_aws.sh`.
36+
37+
The Makefile provides targets that wrap these scripts:
38+
```bash
39+
make build-aws # builds all AWS C libraries
40+
make build-arrow # builds Arrow C++ library only
41+
make build-arrow-glib # builds Arrow C++ with GLib bindings
42+
```
43+
44+
Be sure to run these targets before running `make`, building tests, or contributing code that depends on AWS or Arrow.
45+
46+
## Debugging
47+
48+
To enable detailed debug output when running Embedding Bridge commands, set the following environment variables in your shell:
49+
50+
```sh
51+
EB_DEBUG=1 EB_DEBUG_LEVEL=5 <your-command>
52+
```
53+
54+
- `EB_DEBUG=1` enables debug output.
55+
- `EB_DEBUG_LEVEL` controls verbosity:
56+
- `0` (none, no debug output)
57+
- `1` (error)
58+
- `2` (warn)
59+
- `3` (info)
60+
- `4` (debug)
61+
- `5` (trace, very verbose)
62+
63+
This can help diagnose issues during development or when submitting bug reports.
64+
2965
## License
3066

3167
By contributing, you agree that your contributions will be licensed under the GNU General Public License v2.0.

‎README.md‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ A command-line tool for managing and versioning embedding vectors. Think of it a
1010
- Roll back to previous embedding versions
1111
- Support for multiple embedding models
1212
- Organize embeddings into sets for better management
13-
- Prepare for remote collaboration (coming soon)
1413

1514
## Installation
1615

@@ -20,6 +19,25 @@ make all
2019
make install
2120
```
2221

22+
### From release
23+
24+
```bash
25+
# Download the release archive (tar.gz or zip):
26+
# tar -xzf embedding_bridge-<version>.tar.gz
27+
# # or
28+
# unzip embedding_bridge-<version>.zip
29+
# cd embedding_bridge-<version>
30+
31+
# Create a short-name `embr` launcher in this folder (symlink to the wrapper):
32+
ln -s run_embedding_bridge.sh embr
33+
34+
# Add the release directory to your PATH so you can invoke the launcher:
35+
export PATH="$PWD:$PATH"
36+
37+
# Now run the CLI (the wrapper will set LD_LIBRARY_PATH for you):
38+
embr --help
39+
```
40+
2341
## Quick Start
2442

2543
```bash
@@ -43,7 +61,7 @@ embr rollback <hash> document.txt
4361

4462
# Create and manage sets
4563
embr set create experimental
46-
embr set switch experimental
64+
embr switch experimental
4765
```
4866

4967
## Core Commands
@@ -74,46 +92,68 @@ embr diff <hash1> <hash2>
7492

7593
# Roll back to previous version
7694
embr rollback <hash> file.txt
95+
96+
# View embedding log
97+
embr log file.txt
7798
```
7899

79100
### Set Management
80101
```bash
81102
# Create a new set of embeddings
82-
embr set create <name> [--desc="Description"] [--base=<base-set>]
103+
embr set create <name>
83104

84105
# List available sets
85106
embr set list
86107
embr set list --verbose
87108

88109
# Switch between sets
89-
embr set switch <name>
110+
embr switch <name>
90111

91112
# Show current set status
92113
embr set status
93114

94115
# Compare differences between sets
95116
embr set diff <set1> <set2>
96117

97-
# Merge sets (future functionality)
98-
embr set merge <source-set> [--target=<target-set>] [--strategy=<strategy>]
99-
100118
# Delete a set
101-
embr set delete <name> [--force]
119+
embr set -d <name> [--force]
120+
```
121+
122+
### Merge Sets
123+
```bash
124+
# Merge embeddings from one set to another
125+
embr merge <source-set> [<target-set>] [--strategy=<strategy>]
102126
```
103127

104-
### Remote Operations (Future Functionality)
128+
### Remote Operations
105129
```bash
106130
# Add a remote
107131
embr remote add <name> <url>
108132

109-
# List remote sets
110-
embr remote sets <remote>
133+
# List remotes
134+
embr remote list
111135

112136
# Push a set to remote
113-
embr remote push <set-name> [remote]
137+
embr push <remote> [<set>]
114138

115139
# Pull a set from remote
116-
embr remote pull <set-name> [remote]
140+
embr pull <remote> [<set>]
141+
```
142+
143+
### Other Useful Commands
144+
```bash
145+
# Download a file or directory from a repository
146+
embr get <remote> <path>
147+
148+
# Garbage collect unreferenced embeddings
149+
embr gc [options]
150+
# Example: dry run
151+
enbr gc -n
152+
153+
# Remove embeddings from tracking
154+
embr rm file.txt
155+
embr rm --cached file.txt
156+
embr rm -m openai-3 file.txt
117157
```
118158

119159
## Contributing

‎scripts/build_aws.sh‎

Lines changed: 24 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
set -e
33

4+
git submodule update --init --recursive --remote
5+
46
# Create directory structure
57
mkdir -p vendor/aws/install
68

@@ -9,125 +11,102 @@ INSTALL_DIR=$(pwd)/vendor/aws/install
911

1012
echo "Building AWS libraries in vendor/aws with install directory: $INSTALL_DIR"
1113

12-
cd vendor/aws
13-
1414
# 1. aws-lc (AWS LibCrypto)
15-
if [ ! -d "aws-lc" ]; then
16-
git submodule add -f https://github.com/aws/aws-lc.git
15+
if [ ! -d "vendor/aws/aws-lc" ]; then
16+
git submodule add -f https://github.com/aws/aws-lc.git vendor/aws/aws-lc
1717
else
1818
echo "aws-lc directory already exists, skipping git clone"
1919
fi
20-
cd ..
2120
cmake -S vendor/aws/aws-lc -B vendor/aws/aws-lc/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
2221
cmake --build vendor/aws/aws-lc/build --target install
23-
cd vendor/aws
2422

2523
# 2. s2n-tls
26-
if [ ! -d "s2n-tls" ]; then
27-
git submodule add -f https://github.com/aws/s2n-tls.git
24+
if [ ! -d "vendor/aws/s2n-tls" ]; then
25+
git submodule add -f https://github.com/aws/s2n-tls.git vendor/aws/s2n-tls
2826
else
2927
echo "s2n-tls directory already exists, skipping git clone"
3028
fi
31-
cd ..
3229
cmake -S vendor/aws/s2n-tls -B vendor/aws/s2n-tls/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR
3330
cmake --build vendor/aws/s2n-tls/build --target install
34-
cd vendor/aws
3531

3632
# 3. aws-c-common
37-
if [ ! -d "aws-c-common" ]; then
38-
git submodule add -f https://github.com/awslabs/aws-c-common.git
33+
if [ ! -d "vendor/aws/aws-c-common" ]; then
34+
git submodule add -f https://github.com/awslabs/aws-c-common.git vendor/aws/aws-c-common
3935
else
4036
echo "aws-c-common directory already exists, skipping git clone"
4137
fi
42-
cd ..
4338
cmake -S vendor/aws/aws-c-common -B vendor/aws/aws-c-common/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR
4439
cmake --build vendor/aws/aws-c-common/build --target install
45-
cd vendor/aws
4640

4741
# 4. aws-checksums
48-
if [ ! -d "aws-checksums" ]; then
49-
git submodule add -f https://github.com/awslabs/aws-checksums.git
42+
if [ ! -d "vendor/aws/aws-checksums" ]; then
43+
git submodule add -f https://github.com/awslabs/aws-checksums.git vendor/aws/aws-checksums
5044
else
5145
echo "aws-checksums directory already exists, skipping git clone"
5246
fi
53-
cd ..
5447
cmake -S vendor/aws/aws-checksums -B vendor/aws/aws-checksums/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common
5548
cmake --build vendor/aws/aws-checksums/build --target install
56-
cd vendor/aws
5749

5850
# 5. aws-c-cal
59-
if [ ! -d "aws-c-cal" ]; then
60-
git submodule add -f https://github.com/awslabs/aws-c-cal.git
51+
if [ ! -d "vendor/aws/aws-c-cal" ]; then
52+
git submodule add -f https://github.com/awslabs/aws-c-cal.git vendor/aws/aws-c-cal
6153
else
6254
echo "aws-c-cal directory already exists, skipping git clone"
6355
fi
64-
cd ..
6556
cmake -S vendor/aws/aws-c-cal -B vendor/aws/aws-c-cal/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common
6657
cmake --build vendor/aws/aws-c-cal/build --target install
67-
cd vendor/aws
6858

6959
# 6. aws-c-io
70-
if [ ! -d "aws-c-io" ]; then
71-
git submodule add -f https://github.com/awslabs/aws-c-io.git
60+
if [ ! -d "vendor/aws/aws-c-io" ]; then
61+
git submodule add -f https://github.com/awslabs/aws-c-io.git vendor/aws/aws-c-io
7262
else
7363
echo "aws-c-io directory already exists, skipping git clone"
7464
fi
75-
cd ..
7665
cmake -S vendor/aws/aws-c-io -B vendor/aws/aws-c-io/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common -Daws-c-cal_DIR=$INSTALL_DIR/lib/cmake/aws-c-cal
7766
cmake --build vendor/aws/aws-c-io/build --target install
78-
cd vendor/aws
7967

8068
# 7. aws-c-compression
81-
if [ ! -d "aws-c-compression" ]; then
82-
git submodule add -f https://github.com/awslabs/aws-c-compression.git
69+
if [ ! -d "vendor/aws/aws-c-compression" ]; then
70+
git submodule add -f https://github.com/awslabs/aws-c-compression.git vendor/aws/aws-c-compression
8371
else
8472
echo "aws-c-compression directory already exists, skipping git clone"
8573
fi
86-
cd ..
8774
cmake -S vendor/aws/aws-c-compression -B vendor/aws/aws-c-compression/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common
8875
cmake --build vendor/aws/aws-c-compression/build --target install
89-
cd vendor/aws
9076

9177
# 8. aws-c-http
92-
if [ ! -d "aws-c-http" ]; then
93-
git submodule add -f https://github.com/awslabs/aws-c-http.git
78+
if [ ! -d "vendor/aws/aws-c-http" ]; then
79+
git submodule add -f https://github.com/awslabs/aws-c-http.git vendor/aws/aws-c-http
9480
else
9581
echo "aws-c-http directory already exists, skipping git clone"
9682
fi
97-
cd ..
9883
cmake -S vendor/aws/aws-c-http -B vendor/aws/aws-c-http/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common -Daws-c-compression_DIR=$INSTALL_DIR/lib/cmake/aws-c-compression -Daws-c-io_DIR=$INSTALL_DIR/lib/cmake/aws-c-io
9984
cmake --build vendor/aws/aws-c-http/build --target install
100-
cd vendor/aws
10185

10286
# 9. aws-c-sdkutils
103-
if [ ! -d "aws-c-sdkutils" ]; then
104-
git submodule add -f https://github.com/awslabs/aws-c-sdkutils.git
87+
if [ ! -d "vendor/aws/aws-c-sdkutils" ]; then
88+
git submodule add -f https://github.com/awslabs/aws-c-sdkutils.git vendor/aws/aws-c-sdkutils
10589
else
10690
echo "aws-c-sdkutils directory already exists, skipping git clone"
10791
fi
108-
cd ..
10992
cmake -S vendor/aws/aws-c-sdkutils -B vendor/aws/aws-c-sdkutils/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common
11093
cmake --build vendor/aws/aws-c-sdkutils/build --target install
111-
cd vendor/aws
11294

11395
# 10. aws-c-auth
114-
if [ ! -d "aws-c-auth" ]; then
115-
git submodule add -f https://github.com/awslabs/aws-c-auth.git
96+
if [ ! -d "vendor/aws/aws-c-auth" ]; then
97+
git submodule add -f https://github.com/awslabs/aws-c-auth.git vendor/aws/aws-c-auth
11698
else
11799
echo "aws-c-auth directory already exists, skipping git clone"
118100
fi
119-
cd ..
120101
cmake -S vendor/aws/aws-c-auth -B vendor/aws/aws-c-auth/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR -Daws-c-common_DIR=$INSTALL_DIR/lib/cmake/aws-c-common -Daws-c-cal_DIR=$INSTALL_DIR/lib/cmake/aws-c-cal -Daws-c-io_DIR=$INSTALL_DIR/lib/cmake/aws-c-io -Daws-c-http_DIR=$INSTALL_DIR/lib/cmake/aws-c-http -Daws-c-sdkutils_DIR=$INSTALL_DIR/lib/cmake/aws-c-sdkutils
121102
cmake --build vendor/aws/aws-c-auth/build --target install
122-
cd vendor/aws
123103

124104
# 11. aws-c-s3
125-
if [ ! -d "aws-c-s3" ]; then
126-
git submodule add -f https://github.com/awslabs/aws-c-s3.git
105+
if [ ! -d "vendor/aws/aws-c-s3" ]; then
106+
git submodule add -f https://github.com/awslabs/aws-c-s3.git vendor/aws/aws-c-s3
127107
else
128108
echo "aws-c-s3 directory already exists, skipping git clone"
129109
fi
130-
cd ..
131110
cmake -S vendor/aws/aws-c-s3 -B vendor/aws/aws-c-s3/build -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DCMAKE_PREFIX_PATH=$INSTALL_DIR
132111
cmake --build vendor/aws/aws-c-s3/build --target install
133112

‎src/cli/cli.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ typedef struct {
4949
int cmd_init(int argc, char** argv);
5050
int cmd_store(int argc, char** argv);
5151
int cmd_diff(int argc, char** argv);
52-
int cmd_hooks(int argc, char** argv);
5352
int cmd_config(int argc, char** argv);
5453
int cmd_remote(int argc, char** argv);
5554
int cmd_model(int argc, char** argv);
@@ -62,6 +61,8 @@ int cmd_merge(int argc, char **argv);
6261
int cmd_gc(int argc, char **argv);
6362
int cmd_get(int argc, char **argv);
6463
int cmd_rm(int argc, char **argv);
64+
int cmd_pull(int argc, char **argv);
65+
int cmd_push(int argc, char **argv);
6566

6667
// Option parsing
6768
bool parse_cli_options(int argc, char** argv, eb_cli_options_t* opts);

‎src/cli/diff.c‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -883,10 +883,9 @@ static bool is_hex_string(const char* str) {
883883

884884
/* Check if a file has embeddings from multiple models */
885885
static bool has_multiple_models(const char* repo_root, const char* file_path) {
886-
char log_path[PATH_MAX];
887-
snprintf(log_path, sizeof(log_path), "%s/.embr/log", repo_root);
888-
886+
char* log_path = get_current_set_log_path();
889887
FILE* fp = fopen(log_path, "r");
888+
free(log_path);
890889
if (!fp) return false;
891890

892891
char line[1024];
@@ -960,11 +959,9 @@ static bool has_multiple_models(const char* repo_root, const char* file_path) {
960959
static char* get_available_models(const char* repo_root, const char* file_path) {
961960
static char model_list[512] = {0};
962961
model_list[0] = '\0';
963-
964-
char log_path[PATH_MAX];
965-
snprintf(log_path, sizeof(log_path), "%s/.embr/log", repo_root);
966-
962+
char* log_path = get_current_set_log_path();
967963
FILE* fp = fopen(log_path, "r");
964+
free(log_path);
968965
if (!fp) return model_list;
969966

970967
char line[1024];
@@ -1045,11 +1042,9 @@ static char* get_available_models(const char* repo_root, const char* file_path)
10451042
static const char* get_default_model_for_file(const char* repo_root, const char* file_path) {
10461043
static char default_model[64] = {0};
10471044
default_model[0] = '\0';
1048-
1049-
char log_path[PATH_MAX];
1050-
snprintf(log_path, sizeof(log_path), "%s/.embr/log", repo_root);
1051-
1045+
char* log_path = get_current_set_log_path();
10521046
FILE* fp = fopen(log_path, "r");
1047+
free(log_path);
10531048
if (!fp) return NULL;
10541049

10551050
char line[1024];

0 commit comments

Comments
 (0)