<?xml version="1.0"?>
<ownershipDocument>
    <schemaVersion>X0201</schemaVersion>

    <documentType>4</documentType>
<periodOfReport>2004-01-02</periodOfReport>
<notSubjectToSection16>0</notSubjectToSection16>
    <issuer>
        <issuerCik>0000810689</issuerCik>
        <issuerName>BANK OF GRANITE CORP</issuerName>
        <issuerTradingSymbol>gran</issuerTradingSymbol>
</issuer>
<reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001254274</rptOwnerCik>
            <rptOwnerName>STURGES WESLEY W</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>C/O BANK OF GRANITE</rptOwnerStreet1>
            <rptOwnerStreet2>301 S MCDOWELL STREET #100</rptOwnerStreet2>
            <rptOwnerCity>CHARLOTTE</rptOwnerCity>
            <rptOwnerState>NC</rptOwnerState>
            <rptOwnerZipCode>28204</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
            <reportingOwnerRelationship>
<isDirector>0</isDirector>
<isOfficer>1</isOfficer>
<isTenPercentOwner>0</isTenPercentOwner>
<isOther>0</isOther>
<officerTitle>Executive Vice President</officerTitle>
<otherText></otherText>
</reportingOwnerRelationship>
</reportingOwner>
<nonDerivativeTable>
<nonDerivativeTransaction>
<securityTitle>
<value>Common Stock</value>
</securityTitle>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>7024</value>
</transactionShares>
<transactionPricePerShare>
<value>8.76</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>A</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>21457</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</nonDerivativeTransaction>
<nonDerivativeTransaction>
<securityTitle>
<value>Common Stock</value>
</securityTitle>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>6301</value>
</transactionShares>
<transactionPricePerShare>
<value>10.66</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>A</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>27758</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</nonDerivativeTransaction>
<nonDerivativeTransaction>
<securityTitle>
<value>Common Stock</value>
</securityTitle>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>5936</value>
</transactionShares>
<transactionPricePerShare>
<value>11.93</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>A</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>33694</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</nonDerivativeTransaction>
<nonDerivativeTransaction>
<securityTitle>
<value>Common Stock</value>
</securityTitle>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>4566</value>
</transactionShares>
<transactionPricePerShare>
<value>8.25</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>A</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>38260</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</nonDerivativeTransaction>
<nonDerivativeTransaction>
<securityTitle>
<value>Common Stock</value>
</securityTitle>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>6030</value>
</transactionShares>
<transactionPricePerShare>
<value>7.18</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>A</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>44290</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</nonDerivativeTransaction>
<nonDerivativeTransaction>
<securityTitle>
<value>Common Stock</value>
</securityTitle>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>4618</value>
</transactionShares>
<transactionPricePerShare>
<value>10.96</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>A</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>48908</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</nonDerivativeTransaction>
</nonDerivativeTable>
<derivativeTable>
<derivativeTransaction>
<securityTitle>
<value>Option</value>
</securityTitle>
<conversionOrExercisePrice>
<value>8.76</value>
</conversionOrExercisePrice>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>7024</value>
</transactionShares>
<transactionPricePerShare>
<value>21.88</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>D</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<exerciseDate>
<value>2003-07-15</value>
</exerciseDate>
<expirationDate>
<value>2007-07-14</value>
</expirationDate>
<underlyingSecurity>
<underlyingSecurityTitle>
<value>Common Stock</value>
</underlyingSecurityTitle>
<underlyingSecurityShares>
<value>7024</value>
</underlyingSecurityShares>
</underlyingSecurity>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>0</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</derivativeTransaction>
<derivativeTransaction>
<securityTitle>
<value>Option</value>
</securityTitle>
<conversionOrExercisePrice>
<value>10.66</value>
</conversionOrExercisePrice>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>6301</value>
</transactionShares>
<transactionPricePerShare>
<value>21.88</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>D</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<exerciseDate>
<value>2003-07-15</value>
</exerciseDate>
<expirationDate>
<value>2008-05-05</value>
</expirationDate>
<underlyingSecurity>
<underlyingSecurityTitle>
<value>Common Stock</value>
</underlyingSecurityTitle>
<underlyingSecurityShares>
<value>6301</value>
</underlyingSecurityShares>
</underlyingSecurity>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>0</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</derivativeTransaction>
<derivativeTransaction>
<securityTitle>
<value>Option</value>
</securityTitle>
<conversionOrExercisePrice>
<value>11.93</value>
</conversionOrExercisePrice>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>5936</value>
</transactionShares>
<transactionPricePerShare>
<value>21.88</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>D</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<exerciseDate>
<value>2003-07-15</value>
</exerciseDate>
<expirationDate>
<value>2009-06-25</value>
</expirationDate>
<underlyingSecurity>
<underlyingSecurityTitle>
<value>Common Stock</value>
</underlyingSecurityTitle>
<underlyingSecurityShares>
<value>5936</value>
</underlyingSecurityShares>
</underlyingSecurity>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>0</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</derivativeTransaction>
<derivativeTransaction>
<securityTitle>
<value>Option</value>
</securityTitle>
<conversionOrExercisePrice>
<value>8.25</value>
</conversionOrExercisePrice>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>4566</value>
</transactionShares>
<transactionPricePerShare>
<value>21.88</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>D</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<exerciseDate>
<value>2003-07-15</value>
</exerciseDate>
<expirationDate>
<value>2010-06-27</value>
</expirationDate>
<underlyingSecurity>
<underlyingSecurityTitle>
<value>Common Stock</value>
</underlyingSecurityTitle>
<underlyingSecurityShares>
<value>4566</value>
</underlyingSecurityShares>
</underlyingSecurity>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>0</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</derivativeTransaction>
<derivativeTransaction>
<securityTitle>
<value>Option</value>
</securityTitle>
<conversionOrExercisePrice>
<value>7.18</value>
</conversionOrExercisePrice>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>6030</value>
</transactionShares>
<transactionPricePerShare>
<value>21.88</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>D</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<exerciseDate>
<value>2003-07-15</value>
</exerciseDate>
<expirationDate>
<value>2011-05-09</value>
</expirationDate>
<underlyingSecurity>
<underlyingSecurityTitle>
<value>Common Stock</value>
</underlyingSecurityTitle>
<underlyingSecurityShares>
<value>6030</value>
</underlyingSecurityShares>
</underlyingSecurity>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>0</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</derivativeTransaction>
<derivativeTransaction>
<securityTitle>
<value>Option</value>
</securityTitle>
<conversionOrExercisePrice>
<value>10.96</value>
</conversionOrExercisePrice>
<transactionDate>
<value>2004-01-02</value>
</transactionDate>
<deemedExecutionDate>
</deemedExecutionDate>
<transactionCoding>
<transactionFormType>4</transactionFormType>
<transactionCode>M</transactionCode>
<equitySwapInvolved>0</equitySwapInvolved>
</transactionCoding>
<transactionTimeliness>
<value></value>
</transactionTimeliness>
<transactionAmounts>
<transactionShares>
<value>4618</value>
</transactionShares>
<transactionPricePerShare>
<value>21.88</value>
</transactionPricePerShare>
<transactionAcquiredDisposedCode>
<value>D</value>
</transactionAcquiredDisposedCode>
</transactionAmounts>
<exerciseDate>
<value>2003-07-15</value>
</exerciseDate>
<expirationDate>
<value>2012-05-08</value>
</expirationDate>
<underlyingSecurity>
<underlyingSecurityTitle>
<value>Common Stock</value>
</underlyingSecurityTitle>
<underlyingSecurityShares>
<value>4618</value>
</underlyingSecurityShares>
</underlyingSecurity>
<postTransactionAmounts>
<sharesOwnedFollowingTransaction>
<value>0</value>
</sharesOwnedFollowingTransaction>
</postTransactionAmounts>
<ownershipNature>
<directOrIndirectOwnership>
<value>D</value>
</directOrIndirectOwnership>
<natureOfOwnership>
<value></value>
</natureOfOwnership>
</ownershipNature>
</derivativeTransaction>
</derivativeTable>
<remarks></remarks>
<ownerSignature>
<signatureName>/s/Melodie R. Mathes, Attorney in Fact</signatureName>
<signatureDate>2004-01-06</signatureDate>
</ownerSignature>
</ownershipDocument>

