Streaming

const stream = await model.run('Jack and Jill', { stream: true });
const textStream = stream.pipeThrough(new TextDecoderStream());

for await (const chunk of textStream) {
  console.log(chunk); // Process each chunk of the stream
}

from bytez import Bytez

client = Bytez("YOUR_BYTEZ_KEY_HERE")

model = client.model("Qwen/Qwen2-7B-Instruct")
model.load()

input_text = "Once upon a time there was a beautiful home where"
model_params = {"max_new_tokens": 20, "min_new_tokens": 5, "temperature": 0.5}

stream = model.run(
    input_text,
    stream=True,
    model_params=model_params,
)

for chunk in stream:
    print(f"Output: {chunk}")  # Process each chunk

using Bytez

client = Bytez.init("YOUR_BYTEZ_KEY_HERE")

model = client.model("Qwen/Qwen2-7B-Instruct")
model.load()

input_text = "Once upon a time there was a beautiful home where"

options = Dict(
    "params" => Dict(
        "max_new_tokens" => 20,
        "min_new_tokens" => 5,
        "temperature" => 0.5,
    ),
    "stream" => true,
)

stream = model.run(input_text, options)

while isopen(stream)
    item = take!(stream)  # Take each item as it enters the channel
    println(item)  # Print the item
end