The first thing of note is that you cannot directly use sub-queries in check constraints, but creating a function is the way around that. This will help you:
CREATE FUNCTION dbo.fnRedRows()
RETURNS INT
AS
BEGIN
DECLARE @Return INT
SELECT @Return=COUNT(*) FROM dbo.Red
RETURN @Return
END
GO
CREATE TABLE dbo.Red
(
id INT IDENTITY(1,1),
test VARCHAR(max),
CONSTRAINT CK_MaxRows CHECK (dbo.fnRedRows()<5)
)
GO
INSERT INTO dbo.Red (test) VALUES ('HI')
INSERT INTO dbo.Red (test) VALUES ('The')
INSERT INTO dbo.Red (test) VALUES ('first four')
INSERT INTO dbo.Red (test) VALUES ('will work')
INSERT INTO dbo.Red (test) VALUES ('This one will fail')
GO
DROP TABLE dbo.Red
GO
DROP FUNCTION dbo.fnRedRows
GO