Merge pull request 'develop' (#7) from develop into main

Reviewed-on: KeshavAnandCode/foreign-thon#7

README.md

@@ -6,10 +6,10 @@ ForeignThon transpiles `.es.py`, `.ta.py` (and more) into standard Python. Keywo
 
 ```python
 # hola.es.py
-definir saludar(nombre):
+def saludar(nombre):
     retornar f"Hola, {nombre}!"
 
-para i en rango(3):
+para i en dist(3):
     imprimir(saludar(f"mundo {i}"))
 ```
 

packages/foreignthon/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "foreignthon"
-version = "0.4.1"
+version = "0.5.2"
 description = "Write Python in any language. Transpiles foreign-language .xx.py files to standard Python."
 license = { text = "GPL v3" }
 requires-python = ">=3.9"
@@ -56,3 +56,6 @@ ignore = ["E501"]
 
 [tool.ruff.lint.per-file-ignores]
 "pack.py" = ["E501"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"src/foreignthon/template.json" = "foreignthon/template.json"

packages/foreignthon/src/foreignthon/cli.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import subprocess
 import sys
 from pathlib import Path
 
@@ -11,6 +12,72 @@ from .transpiler import run_transpiled, transpile_file
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
+def _load_effective_pack(project: Path, lang: str) -> dict:
+    """
+    Load pack for a file. Priority:
+    1. custom_pack in .foreignthon.toml — no installed pack needed
+    2. installed pack via entry points
+    """
+    import json
+
+    from .pack import load_pack
+
+    # Walk up to find .foreignthon.toml
+    search = project if project.is_dir() else project.parent
+    toml_path = None
+    for parent in [search, *search.parents]:
+        candidate = parent / ".foreignthon.toml"
+        if candidate.exists():
+            toml_path = candidate
+            break
+
+    # Check for custom_pack first — if found, load it directly
+    # and merge on top of template (no installed pack required)
+    if toml_path:
+        for line in toml_path.read_text(encoding="utf-8").splitlines():
+            if line.strip().startswith("custom_pack") and not line.strip().startswith("#"):
+                custom_path = toml_path.parent / line.split("=", 1)[1].strip().strip('"').strip("'")
+                if custom_path.exists():
+                    custom = json.loads(custom_path.read_text(encoding="utf-8"))
+                    # If custom pack has meta with a code, it's a standalone pack
+                    if custom.get("meta", {}).get("code"):
+                        return custom
+                    # Otherwise it's an override — merge on top of installed pack
+                    pack = load_pack(lang)
+                    for section in ("keywords", "builtins", "exceptions", "stdlib", "error_messages"):
+                        if section in custom:
+                            pack[section] = {**pack.get(section, {}), **custom[section]}
+                    if "postfix_keywords" in custom:
+                        pack["postfix_keywords"] = custom["postfix_keywords"]
+                    return pack
+                break
+
+    # Fall back to installed pack
+    return load_pack(lang)
+
+
+def _pick(mapping: dict, english: str, fallback: str) -> str:
+    reverse = {v: k for k, v in mapping.items()}
+    return reverse.get(english, fallback)
+
+
+def _git_init(project: Path) -> None:
+    git_ok = subprocess.run(["git", "--version"], capture_output=True).returncode == 0
+    if git_ok:
+        subprocess.run(["git", "init"], cwd=project, capture_output=True)
+        subprocess.run(["git", "add", "."], cwd=project, capture_output=True)
+        subprocess.run(
+            ["git", "commit", "-m", "initial commit"],
+            cwd=project, capture_output=True,
+        )
+
+
+def _lang_from_file(path: Path) -> str:
+    suffixes = path.suffixes
+    if len(suffixes) >= 2 and suffixes[-1] == ".py":
+        return suffixes[-2].lstrip(".")
+    return "en"
+
 
 @click.group(context_settings=CONTEXT_SETTINGS)
 @click.version_option(__version__, prog_name="fpy")
@@ -19,6 +86,115 @@ def main():
     pass
 
 
+@main.command(context_settings=CONTEXT_SETTINGS)
+@click.argument("name", default="")
+@click.option("--lang", "-l", required=True, help="Language code (e.g. es, ta)")
+@click.option("--no-git", is_flag=True, help="Skip git init")
+def new(name: str, lang: str, no_git: bool):
+    """
+    Create a new ForeignThon project.
+
+    \b
+    fpy new myproject --lang es     # creates myproject/
+    fpy new --lang es               # initializes current directory
+    """
+    from .pack import PackNotFoundError
+
+    if name:
+        project = Path(name)
+        if project.exists():
+            click.echo(f"✗ '{name}' already exists.", err=True)
+            raise SystemExit(1)
+        project.mkdir(parents=True)
+    else:
+        project = Path.cwd()
+        if any(project.iterdir()):
+            click.echo("✗ Current directory is not empty.", err=True)
+            raise SystemExit(1)
+
+    is_custom = lang == "custom"
+
+    if is_custom:
+        lang_code = click.prompt("Language code (e.g. ru, fr, de)")
+        lang_name_en = click.prompt("Language name in English (e.g. Russian)")
+        lang_name_native = click.prompt("Language name in its own script (e.g. Русский)")
+        pack = _make_scaffold_pack(lang_code, lang_name_en, lang_name_native)
+        lang = lang_code
+    else:
+        try:
+            pack = _load_effective_pack(project, lang)
+        except PackNotFoundError:
+            click.echo(f"✗ Language pack '{lang}' not installed.", err=True)
+            click.echo(f"  Run: pip install foreignthon-{lang}", err=True)
+            raise SystemExit(1)
+
+    lang_name = pack["meta"]["native_name"]
+
+    if is_custom:
+        import json
+        (project / "custom.json").write_text(
+            json.dumps(pack, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        toml_custom_line = 'custom_pack = "custom.json"'
+        click.echo("  Created custom.json — fill in your translations!")
+    else:
+        toml_custom_line = '# custom_pack = "custom.json"'
+
+    (project / ".foreignthon.toml").write_text(
+        f'[foreignthon]\n'
+        f'lang = "{lang}"\n'
+        f'\n'
+        f'# Optional: path to a local JSON that overrides pack keywords\n'
+        f'{toml_custom_line}\n',
+        encoding="utf-8",
+    )
+
+    (project / ".gitignore").write_text(
+        "__pycache__/\n"
+        "*.py[cod]\n"
+        "*.egg-info/\n"
+        "dist/\n"
+        "build/\n"
+        ".venv/\n"
+        ".pytest_cache/\n"
+        ".DS_Store\n"
+        "Thumbs.db\n",
+        encoding="utf-8",
+    )
+
+    src = project / "src"
+    src.mkdir(exist_ok=True)
+
+    bi_print = _pick(pack["builtins"], "print", "print")
+    (src / f"main.{lang}.py").write_text(
+        f"# Hello, World! in {lang_name}\n"
+        f'{bi_print}("Hello, World!")\n',
+        encoding="utf-8",
+    )
+
+    (project / "README.md").write_text(
+        f"# {name or project.name}\n\n"
+        f"A ForeignThon project in {lang_name}.\n\n"
+        f"## Run\n\n"
+        f"```bash\n"
+        f"fpy run src/main.{lang}.py\n"
+        f"```\n\n"
+        f"## Custom pack override\n\n"
+        f"Create a `custom.json` and set `custom_pack = \"custom.json\"` "
+        f"in `.foreignthon.toml` to add or override keywords locally.\n",
+        encoding="utf-8",
+    )
+
+    if not no_git:
+        _git_init(project)
+
+    click.echo(f"✓ Created '{name or '.'}' [{lang_name}]")
+    if name:
+        click.echo(f"  cd {name}")
+    click.echo(f"  fpy run src/main.{lang}.py")
+
+
 @main.command(context_settings=CONTEXT_SETTINGS)
 @click.argument("file", type=click.Path(exists=True, path_type=Path))
 @click.option("--lang", "-l", default=None, help="Override language code (e.g. es, ta)")
@@ -34,7 +210,8 @@ def run(file: Path, lang: str | None, keep: bool):
     detected_lang = lang or _lang_from_file(file)
     activate(detected_lang)
 
-    transpiled = transpile_file(file)
+    pack = _load_effective_pack(file, detected_lang)
+    transpiled = transpile_file(file, pack=pack)
 
     if keep:
         out_path = file.with_suffix("").with_suffix(".compiled.py")
@@ -54,14 +231,14 @@ def compile(file: Path, output: str | None):
     """
     Transpile a foreign-language file to standard Python.
 
-    Output can be a file path or a directory:
-
     \b
     fpy compile script.es.py              # → script.compiled.py
     fpy compile script.es.py -o out/      # → out/script.compiled.py
     fpy compile script.es.py -o out.py    # → out.py
     """
-    transpiled = transpile_file(file)
+    detected_lang = _lang_from_file(file)
+    pack = _load_effective_pack(file, detected_lang)
+    transpiled = transpile_file(file, pack=pack)
 
     if output is None:
         out_path = file.with_suffix("").with_suffix(".compiled.py")
@@ -76,26 +253,60 @@ def compile(file: Path, output: str | None):
     out_path.parent.mkdir(parents=True, exist_ok=True)
     out_path.write_text(transpiled, encoding="utf-8")
     click.echo(f"Compiled: {out_path}")
+    
+@main.command(context_settings=CONTEXT_SETTINGS)
+@click.argument("files", nargs=-1, required=True, type=click.Path(exists=True, path_type=Path))
+def check(files: tuple):
+    """Validate one or more foreign-language files without running them."""
+    import ast
+    import io
+    import tokenize as _tokenize
 
+    from .transpiler import _build_mapping, transpile_file
+
+    failed = False
+    for file in files:
+        detected_lang = _lang_from_file(file)
+        try:
+            pack = _load_effective_pack(file, detected_lang)
+            transpiled = transpile_file(file, pack=pack)
+            ast.parse(transpiled)
+
+            source = file.read_text(encoding="utf-8")
+            mapping = _build_mapping(pack)
+            tokens = list(_tokenize.generate_tokens(io.StringIO(source).readline))
+            count = sum(1 for t in tokens if t.type == _tokenize.NAME and t.string in mapping)
+
+            click.echo(f"✓ {file.name} looks good. ({count} tokens translated)")
+        except SyntaxError as e:
+            click.echo(f"✗ {file.name}: Syntax error: {e}", err=True)
+            failed = True
+        except Exception as e:
+            click.echo(f"✗ {file.name}: {e}", err=True)
+            failed = True
+
+    if failed:
+        sys.exit(1)
 
 @main.command(context_settings=CONTEXT_SETTINGS)
-@click.argument("file", type=click.Path(exists=True, path_type=Path))
-def check(file: Path):
-    """Validate a foreign-language file without running it."""
-    import ast
-
-    try:
-        transpiled = transpile_file(file)
-        ast.parse(transpiled)
-        click.echo(f"✓ {file.name} looks good.")
-    except SyntaxError as e:
-        click.echo(f"✗ Syntax error: {e}", err=True)
-        sys.exit(1)
-    except Exception as e:
-        click.echo(f"✗ {e}", err=True)
-        sys.exit(1)
+def langs():
+    """List all installed language packs."""
+    from .pack import _discover_packs
 
+    packs = _discover_packs()
+    if not packs:
+        click.echo("No language packs installed.")
+        click.echo("Try: pip install foreignthon-es")
+        return
 
+    click.echo("Installed language packs:")
+    for code, module in sorted(packs.items()):
+        import json
+        data = json.loads(module.get_pack_path().read_text(encoding="utf-8"))
+        name = data["meta"].get("name", code)
+        native = data["meta"].get("native_name", "")
+        click.echo(f"  {code:<6} {name} ({native})")
+        
 @main.command("pack", context_settings=CONTEXT_SETTINGS)
 @click.argument("json_file", type=click.Path(exists=True, path_type=Path))
 def validate_pack(json_file: Path):
@@ -115,7 +326,6 @@ def validate_pack(json_file: Path):
     click.echo(f"✓ Pack '{data['meta']['name']}' is valid.")
 
 
-
 @main.command(context_settings=CONTEXT_SETTINGS)
 @click.argument("file", type=click.Path(exists=True, path_type=Path))
 @click.option("--lang", "-l", required=True, help="Target language code (e.g. es, ta)")
@@ -125,8 +335,6 @@ def decompile(file: Path, lang: str, postfix: bool, output: str | None):
     """
     Convert standard Python back to a foreign language.
 
-    Keywords and builtins are translated. Variable names are untouched.
-
     \b
     fpy decompile script.py --lang es
     fpy decompile script.py --lang ta --postfix
@@ -134,7 +342,8 @@ def decompile(file: Path, lang: str, postfix: bool, output: str | None):
     """
     from .transpiler import detranspile_file
 
-    result = detranspile_file(file, lang, postfix=postfix)
+    pack = _load_effective_pack(file, lang)
+    result = detranspile_file(file, lang, postfix=postfix, pack=pack)
 
     ext = f".{lang}.py"
 
@@ -154,9 +363,17 @@ def decompile(file: Path, lang: str, postfix: bool, output: str | None):
     click.echo(f"Decompiled: {out_path}")
 
 
-def _lang_from_file(path: Path) -> str:
-    suffixes = path.suffixes
-    if len(suffixes) >= 2 and suffixes[-1] == ".py":
-        return suffixes[-2].lstrip(".")
-    return "en"
+def _make_scaffold_pack(lang_code: str, lang_name: str, native_name: str) -> dict:
+    """
+    Load template.json — single source of truth for all pack keys.
+    To add new keywords/builtins, edit template.json only.
+    """
+    import json
+    from importlib.resources import files
 
+    template_path = files("foreignthon") / "template.json"
+    pack = json.loads(template_path.read_text(encoding="utf-8"))
+    pack["meta"]["name"] = lang_name
+    pack["meta"]["native_name"] = native_name
+    pack["meta"]["code"] = lang_code
+    return pack

packages/foreignthon/src/foreignthon/template.json

@@ -0,0 +1,146 @@
+{
+  "meta": {
+    "name": "",
+    "native_name": "",
+    "code": "",
+    "version": "0.1.0",
+    "authors": []
+  },
+  "keywords": {
+    "if": "if",
+    "else": "else",
+    "elif": "elif",
+    "for": "for",
+    "while": "while",
+    "def": "def",
+    "class": "class",
+    "import": "import",
+    "from": "from",
+    "as": "as",
+    "return": "return",
+    "break": "break",
+    "continue": "continue",
+    "pass": "pass",
+    "try": "try",
+    "except": "except",
+    "finally": "finally",
+    "raise": "raise",
+    "with": "with",
+    "in": "in",
+    "is": "is",
+    "and": "and",
+    "or": "or",
+    "not": "not",
+    "del": "del",
+    "global": "global",
+    "nonlocal": "nonlocal",
+    "assert": "assert",
+    "yield": "yield",
+    "await": "await",
+    "async": "async",
+    "lambda": "lambda",
+    "True": "True",
+    "False": "False",
+    "None": "None"
+  },
+  "builtins": {
+    "print": "print",
+    "input": "input",
+    "len": "len",
+    "range": "range",
+    "type": "type",
+    "int": "int",
+    "float": "float",
+    "str": "str",
+    "list": "list",
+    "dict": "dict",
+    "set": "set",
+    "tuple": "tuple",
+    "bool": "bool",
+    "open": "open",
+    "enumerate": "enumerate",
+    "map": "map",
+    "filter": "filter",
+    "sorted": "sorted",
+    "reversed": "reversed",
+    "sum": "sum",
+    "min": "min",
+    "max": "max",
+    "abs": "abs",
+    "round": "round",
+    "all": "all",
+    "any": "any",
+    "isinstance": "isinstance",
+    "hasattr": "hasattr",
+    "getattr": "getattr",
+    "setattr": "setattr",
+    "repr": "repr",
+    "format": "format",
+    "vars": "vars",
+    "next": "next",
+    "id": "id",
+    "chr": "chr",
+    "hex": "hex",
+    "bin": "bin",
+    "oct": "oct"
+  },
+  "exceptions": {
+    "Exception": "Exception",
+    "BaseException": "BaseException",
+    "ValueError": "ValueError",
+    "TypeError": "TypeError",
+    "KeyError": "KeyError",
+    "IndexError": "IndexError",
+    "AttributeError": "AttributeError",
+    "NameError": "NameError",
+    "ImportError": "ImportError",
+    "OSError": "OSError",
+    "FileNotFoundError": "FileNotFoundError",
+    "RuntimeError": "RuntimeError",
+    "StopIteration": "StopIteration",
+    "SystemExit": "SystemExit",
+    "KeyboardInterrupt": "KeyboardInterrupt",
+    "NotImplementedError": "NotImplementedError",
+    "ZeroDivisionError": "ZeroDivisionError",
+    "RecursionError": "RecursionError",
+    "SyntaxError": "SyntaxError",
+    "AssertionError": "AssertionError",
+    "OverflowError": "OverflowError",
+    "MemoryError": "MemoryError",
+    "PermissionError": "PermissionError",
+    "TimeoutError": "TimeoutError"
+  },
+  "error_messages": {
+    "SyntaxError": "SyntaxError",
+    "ValueError": "ValueError",
+    "TypeError": "TypeError",
+    "KeyError": "KeyError",
+    "IndexError": "IndexError",
+    "AttributeError": "AttributeError",
+    "NameError": "NameError",
+    "ImportError": "ImportError",
+    "FileNotFoundError": "FileNotFoundError",
+    "ZeroDivisionError": "ZeroDivisionError",
+    "RecursionError": "RecursionError",
+    "RuntimeError": "RuntimeError",
+    "MemoryError": "MemoryError",
+    "OverflowError": "OverflowError",
+    "AssertionError": "AssertionError",
+    "NotImplementedError": "NotImplementedError",
+    "StopIteration": "StopIteration",
+    "KeyboardInterrupt": "KeyboardInterrupt",
+    "PermissionError": "PermissionError",
+    "TimeoutError": "TimeoutError"
+  },
+  "stdlib": {
+    "math": "math",
+    "sys": "sys",
+    "datetime": "datetime",
+    "time": "time",
+    "random": "random",
+    "collections": "collections",
+    "pathlib": "pathlib",
+    "re": "re"
+  },
+  "postfix_keywords": []
+}

packages/foreignthon/src/foreignthon/transpiler.py

@@ -63,7 +63,6 @@ def _apply_postfix_output(source: str, en_to_foreign: dict, postfix_english: set
 
 
 def _get_slice(source_lines: list[str], sr: int, sc: int, er: int, ec: int) -> str:
-    """Extract text from source between two (row, col) positions (1-indexed rows)."""
     n = len(source_lines)
     if sr > n:
         return ""
@@ -81,10 +80,6 @@ def _get_slice(source_lines: list[str], sr: int, sc: int, er: int, ec: int) -> s
 
 
 def _swap_tokens(source: str, mapping: dict) -> str:
-    """
-    Swap NAME tokens while copying all inter-token text verbatim from source.
-    This preserves original spacing exactly — no double newlines, no extra spaces.
-    """
     source_lines = source.splitlines(keepends=True)
     tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))
 
@@ -96,12 +91,9 @@ def _swap_tokens(source: str, mapping: dict) -> str:
             break
 
         s_row, s_col = tok_start
-
-        # Copy original whitespace/newlines between tokens verbatim
         gap = _get_slice(source_lines, prev_end[0], prev_end[1], s_row, s_col)
         result.append(gap)
 
-        # Swap or keep token
         if tok_type == tokenize.NAME and tok_string in mapping:
             result.append(mapping[tok_string])
         else:
@@ -112,21 +104,26 @@ def _swap_tokens(source: str, mapping: dict) -> str:
     return "".join(result)
 
 
-def transpile(source: str, lang_code: str) -> str:
-    pack = load_pack(lang_code)
-
+def _build_mapping(pack: dict) -> dict:
     mapping: dict[str, str] = {}
     mapping.update(pack["keywords"])
     mapping.update(pack["builtins"])
     mapping.update(pack["exceptions"])
     mapping.update(pack["stdlib"])
+    return mapping
 
+
+def transpile(source: str, lang_code: str, pack: dict | None = None) -> str:
+    if pack is None:
+        pack = load_pack(lang_code)
+    mapping = _build_mapping(pack)
     source = _apply_postfix_syntax(source, mapping)
     return _swap_tokens(source, mapping)
 
 
-def detranspile(source: str, lang_code: str, postfix: bool = False) -> str:
-    pack = load_pack(lang_code)
+def detranspile(source: str, lang_code: str, postfix: bool = False, pack: dict | None = None) -> str:
+    if pack is None:
+        pack = load_pack(lang_code)
 
     en_to_foreign: dict[str, str] = {}
     for section in ("keywords", "builtins", "exceptions", "stdlib"):
@@ -142,16 +139,16 @@ def detranspile(source: str, lang_code: str, postfix: bool = False) -> str:
     return output
 
 
-def transpile_file(path: Path) -> str:
+def transpile_file(path: Path, pack: dict | None = None) -> str:
     lang_code = _detect_lang(path)
     source = path.read_text(encoding="utf-8")
     lang_code = _check_shebang(source, lang_code)
-    return transpile(source, lang_code)
+    return transpile(source, lang_code, pack=pack)
 
 
-def detranspile_file(path: Path, lang_code: str, postfix: bool = False) -> str:
+def detranspile_file(path: Path, lang_code: str, postfix: bool = False, pack: dict | None = None) -> str:
     source = path.read_text(encoding="utf-8")
-    return detranspile(source, lang_code, postfix=postfix)
+    return detranspile(source, lang_code, postfix=postfix, pack=pack)
 
 
 def run_transpiled(original_path: Path, transpiled: str) -> None:

packages/langs/es/pyproject.toml

@@ -14,7 +14,7 @@ authors = [
 ]
 keywords = ["foreignthon", "spanish", "español"]
 
-dependencies = ["foreignthon>=0.4.1"]
+dependencies = ["foreignthon>=0.5.2"]
 
 [project.entry-points."foreignthon.langs"]
 es = "foreignthon_es"

packages/langs/ta/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
   { name = "Keshav Anand", email = "keshavanand.dev@gmail.com" }
 ]
 keywords = ["foreignthon", "tamil", "தமிழ்"]
-dependencies = ["foreignthon>=0.4.1"]
+dependencies = ["foreignthon>=0.5.2"]
 
 [project.entry-points."foreignthon.langs"]
 ta = "foreignthon_ta"