文字列でこれを行うことができます。
CREATE TABLE T
( NUM NUMERIC(38,19) );
INSERT INTO T (NUM) VALUES ( 123456789 );
INSERT INTO T (NUM) VALUES ( 12345 );
INSERT INTO T (NUM) VALUES ( 123 );
INSERT INTO T (NUM) VALUES ( 1 );
SELECT CAST(
CASE WHEN NUM < 999 THEN CAST(FLOOR(NUM) AS VARCHAR)
ELSE SUBSTRING(CAST(NUM AS VARCHAR), 1, 3) + '.'
+ SUBSTRING(CAST(FLOOR(NUM) AS VARCHAR), 4, LEN(CAST(NUM AS VARCHAR)) - 3)
END AS NUMERIC(38, 19))
FROM T
UPDATE T
SET NUM = CAST(CASE WHEN NUM < 999 THEN CAST(FLOOR(NUM) AS VARCHAR)
ELSE SUBSTRING(CAST(NUM AS VARCHAR), 1, 3) + '.'
+ SUBSTRING(CAST(FLOOR(NUM) AS VARCHAR), 4, LEN(CAST(NUM AS VARCHAR)) - 3)
END AS NUMERIC(38, 19));
SQLFiddle に実際の例を載せました。