Skip to content

Commit c73d0bb

Browse files
committed
refactor(scripts): improve count-tokens.sh robustness (#364)
- Add CI/CD detection to skip interactive prompts - Check file existence before counting tokens - Guard against division by zero in savings calculation - Suppress blake2b/blake2s errors for clean output
1 parent 96238c9 commit c73d0bb

1 file changed

Lines changed: 51 additions & 14 deletions

File tree

scripts/count-tokens.sh

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,16 @@ if command -v python3 &> /dev/null; then
1919
# tiktoken not found - offer to install
2020
echo "⚠️ tiktoken not installed"
2121
echo ""
22-
echo "tiktoken provides accurate token counting for Claude/GPT models."
23-
read -p "📦 Install tiktoken now? (y/n): " -n 1 -r
24-
echo ""
25-
if [[ $REPLY =~ ^[Yy]$ ]]; then
26-
echo "📥 Installing tiktoken..."
22+
23+
# Detect non-interactive environment (CI/CD)
24+
if [ ! -t 0 ] || [ -n "$CI" ] || [ -n "$CI_CD" ]; then
25+
echo "🤖 Non-interactive environment detected (CI/CD)"
26+
echo "📝 Using word-based approximation"
27+
echo " (To auto-install in CI, set AUTO_INSTALL_TIKTOKEN=1)"
28+
echo ""
29+
USE_APPROX=1
30+
elif [ -n "$AUTO_INSTALL_TIKTOKEN" ]; then
31+
echo "📥 Installing tiktoken (AUTO_INSTALL_TIKTOKEN=1)..."
2732
if pip3 install tiktoken --quiet; then
2833
echo "✅ tiktoken installed successfully!"
2934
echo ""
@@ -35,10 +40,27 @@ if command -v python3 &> /dev/null; then
3540
USE_APPROX=1
3641
fi
3742
else
38-
echo "📝 Using word-based approximation instead"
39-
echo " (Install manually: pip3 install tiktoken)"
43+
echo "tiktoken provides accurate token counting for Claude/GPT models."
44+
read -p "📦 Install tiktoken now? (y/n): " -n 1 -r
4045
echo ""
41-
USE_APPROX=1
46+
if [[ $REPLY =~ ^[Yy]$ ]]; then
47+
echo "📥 Installing tiktoken..."
48+
if pip3 install tiktoken --quiet; then
49+
echo "✅ tiktoken installed successfully!"
50+
echo ""
51+
# Re-run the script after installation
52+
exec "$0" "$@"
53+
else
54+
echo "❌ Installation failed. Using word-based approximation instead."
55+
echo ""
56+
USE_APPROX=1
57+
fi
58+
else
59+
echo "📝 Using word-based approximation instead"
60+
echo " (Install manually: pip3 install tiktoken)"
61+
echo ""
62+
USE_APPROX=1
63+
fi
4264
fi
4365
fi
4466

@@ -63,13 +85,23 @@ PYTHON
6385

6486
# Count tokens for each file
6587
echo "📄 .github/copilot-instructions.md"
66-
COPILOT_TOKENS=$(python3 /tmp/count_tokens.py .github/copilot-instructions.md)
67-
echo " Tokens: $COPILOT_TOKENS"
88+
if [ -f ".github/copilot-instructions.md" ]; then
89+
COPILOT_TOKENS=$(python3 /tmp/count_tokens.py .github/copilot-instructions.md 2>&1 | grep -v "ERROR:root:code for hash" | tail -1)
90+
echo " Tokens: $COPILOT_TOKENS"
91+
else
92+
echo " ⚠️ File not found, skipping"
93+
COPILOT_TOKENS=0
94+
fi
6895
echo ""
6996

7097
echo "📄 AGENTS.md"
71-
AGENTS_TOKENS=$(python3 /tmp/count_tokens.py AGENTS.md)
72-
echo " Tokens: $AGENTS_TOKENS"
98+
if [ -f "AGENTS.md" ]; then
99+
AGENTS_TOKENS=$(python3 /tmp/count_tokens.py AGENTS.md 2>&1 | grep -v "ERROR:root:code for hash" | tail -1)
100+
echo " Tokens: $AGENTS_TOKENS"
101+
else
102+
echo " ⚠️ File not found, skipping"
103+
AGENTS_TOKENS=0
104+
fi
73105
echo ""
74106

75107
# Calculate total
@@ -91,8 +123,13 @@ PYTHON
91123
echo "❌ copilot-instructions.md exceeds limit! Optimization required."
92124
fi
93125

94-
SAVINGS=$((AGENTS_TOKENS * 100 / TOTAL))
95-
echo "💡 Savings: ${SAVINGS}% saved when AGENTS.md not needed"
126+
# Calculate savings (guard against division by zero)
127+
if [ $TOTAL -gt 0 ]; then
128+
SAVINGS=$((AGENTS_TOKENS * 100 / TOTAL))
129+
echo "💡 Savings: ${SAVINGS}% saved when AGENTS.md not needed"
130+
else
131+
echo "💡 Savings: 0% (no tokens to count)"
132+
fi
96133

97134
# Cleanup
98135
rm /tmp/count_tokens.py

0 commit comments

Comments
 (0)