Skip to content

A ~9M parameter LLM that talks like a small fish.

Python

Add --prompt for chat

Philip Whited7ee72d0e4cc
diff --git a/README.md b/README.md
index 9ba8654..78a2c4f 100644
--- a/README.md
+++ b/README.md
@@ -122,6 +122,14 @@ You> it is raining outside
Guppy> i think rain is the best thing about outside.
```
+In interactive chat mode, the conversation grows and quickly runs into the 128-token limit, reducing quality.
+You can also invoke chat with a single prompt, and exit after the response:
+
+```bash
+python -m guppylm chat --prompt "tell me a joke"
+```
+
+
---
## Dataset
diff --git a/guppylm/__main__.py b/guppylm/__main__.py
index 1d1f7f8..edbd42d 100644
--- a/guppylm/__main__.py
+++ b/guppylm/__main__.py
@@ -25,20 +25,8 @@ def main():
train()
elif cmd == "chat":
- from .inference import GuppyInference
- engine = GuppyInference("checkpoints/best_model.pt", "data/tokenizer.json")
- print("\nGuppy Chat (type 'quit' to exit)")
- msgs = []
- while True:
- inp = input("\nYou> ").strip()
- if inp.lower() in ("quit", "exit", "q"):
- break
- msgs.append({"role": "user", "content": inp})
- r = engine.chat_completion(msgs)
- msg = r["choices"][0]["message"]
- if msg.get("content"):
- print(f"Guppy> {msg['content']}")
- msgs.append(msg)
+ from .inference import main
+ main()
else:
print(f"Unknown command: {cmd}")
diff --git a/guppylm/inference.py b/guppylm/inference.py
index 6bf2976..86a444a 100644
--- a/guppylm/inference.py
+++ b/guppylm/inference.py
@@ -103,9 +103,16 @@ def main():
p.add_argument("--checkpoint", default="checkpoints/best_model.pt")
p.add_argument("--tokenizer", default="data/tokenizer.json")
p.add_argument("--device", default="cpu")
+ p.add_argument("--prompt", "-p", help="Single prompt mode: ask one question and exit")
args = p.parse_args()
engine = GuppyInference(args.checkpoint, args.tokenizer, args.device)
+
+ if args.prompt:
+ result = engine.chat_completion([{"role": "user", "content": args.prompt}])
+ print(result["choices"][0]["message"]["content"])
+ return
+
print("\nGuppy Chat (type 'quit' to exit)")
msgs = []
while True: