Try
CREATE FUNCTION dbo.RoleCheckUser
(
@UserName sysname,
@RoleName sysname
)
RETURNS BIT
AS
BEGIN
DECLARE @RetVal BIT
SET @RetVal = 0

SELECT @RetVal = 1
WHERE EXISTS
(
SELECT *
FROM sysmembers membs
JOIN sysusers users on membs.memberuid = users.uid
JOIN sysusers groups on membs.groupuid = groups.uid
WHERE
users.name = @UserName
AND groups.name = @RoleName
)

RETURN @RetVal
END
GO

-- Syntax to use the created function
SELECT dbo.RoleCheckUser('dbo', 'db_owner')
GO